summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRyan Beisner <ryan.beisner@canonical.com>2017-11-22 18:24:15 +0000
committerDavid Ames <david.ames@canonical.com>2017-11-27 15:49:38 -0800
commit87bbbe1bb0ebddd24249a8a21111f839b65fd47e (patch)
tree2378a49c6447c9bb84c6b268cae35f47f886c1ee
parentb0964fb4df4056040cd09ed3e27fa3b4ca2eecaf (diff)
Sync charm-helpers
Notes
Notes (review): Verified+1: Canonical CI <uosci-testing-bot@ubuntu.com> Code-Review+2: Chris MacNaughton <chris.macnaughton@canonical.com> Workflow+1: Chris MacNaughton <chris.macnaughton@canonical.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Tue, 28 Nov 2017 08:30:53 +0000 Reviewed-on: https://review.openstack.org/522332 Project: openstack/charm-cinder-backup Branch: refs/heads/master
-rw-r--r--hooks/charmhelpers/contrib/hahelpers/apache.py2
-rw-r--r--hooks/charmhelpers/contrib/hahelpers/cluster.py30
-rw-r--r--hooks/charmhelpers/contrib/network/ip.py4
-rw-r--r--hooks/charmhelpers/contrib/openstack/alternatives.py13
-rw-r--r--hooks/charmhelpers/contrib/openstack/amulet/deployment.py44
-rw-r--r--hooks/charmhelpers/contrib/openstack/amulet/utils.py36
-rw-r--r--hooks/charmhelpers/contrib/openstack/context.py48
-rwxr-xr-xhooks/charmhelpers/contrib/openstack/files/check_haproxy.sh2
-rw-r--r--hooks/charmhelpers/contrib/openstack/ha/utils.py11
-rw-r--r--hooks/charmhelpers/contrib/openstack/neutron.py61
-rw-r--r--hooks/charmhelpers/contrib/openstack/templates/ceph.conf4
-rw-r--r--hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg2
-rw-r--r--hooks/charmhelpers/contrib/openstack/templates/section-oslo-cache6
-rw-r--r--hooks/charmhelpers/contrib/openstack/templating.py2
-rw-r--r--hooks/charmhelpers/contrib/openstack/utils.py42
-rw-r--r--hooks/charmhelpers/contrib/storage/linux/ceph.py42
-rw-r--r--hooks/charmhelpers/contrib/storage/linux/lvm.py8
-rw-r--r--hooks/charmhelpers/contrib/storage/linux/utils.py2
-rw-r--r--hooks/charmhelpers/core/hookenv.py112
-rw-r--r--hooks/charmhelpers/core/host.py73
-rw-r--r--hooks/charmhelpers/core/strutils.py16
-rw-r--r--hooks/charmhelpers/core/unitdata.py2
-rw-r--r--hooks/charmhelpers/fetch/snap.py16
-rw-r--r--hooks/charmhelpers/fetch/ubuntu.py2
-rw-r--r--tests/basic_deployment.py6
-rw-r--r--tests/charmhelpers/contrib/openstack/amulet/deployment.py44
-rw-r--r--tests/charmhelpers/contrib/openstack/amulet/utils.py36
-rw-r--r--tests/charmhelpers/core/hookenv.py112
-rw-r--r--tests/charmhelpers/core/host.py73
-rw-r--r--tests/charmhelpers/core/strutils.py16
-rw-r--r--tests/charmhelpers/core/unitdata.py2
31 files changed, 682 insertions, 187 deletions
diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py
index d0c6994..22acb68 100644
--- a/hooks/charmhelpers/contrib/hahelpers/apache.py
+++ b/hooks/charmhelpers/contrib/hahelpers/apache.py
@@ -90,6 +90,6 @@ def install_ca_cert(ca_cert):
90 log("CA cert is the same as installed version", level=INFO) 90 log("CA cert is the same as installed version", level=INFO)
91 else: 91 else:
92 log("Installing new CA cert", level=INFO) 92 log("Installing new CA cert", level=INFO)
93 with open(cert_file, 'w') as crt: 93 with open(cert_file, 'wb') as crt:
94 crt.write(ca_cert) 94 crt.write(ca_cert)
95 subprocess.check_call(['update-ca-certificates', '--fresh']) 95 subprocess.check_call(['update-ca-certificates', '--fresh'])
diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py
index e02350e..4207e42 100644
--- a/hooks/charmhelpers/contrib/hahelpers/cluster.py
+++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py
@@ -27,6 +27,7 @@ clustering-related helpers.
27 27
28import subprocess 28import subprocess
29import os 29import os
30import time
30 31
31from socket import gethostname as get_unit_hostname 32from socket import gethostname as get_unit_hostname
32 33
@@ -45,6 +46,9 @@ from charmhelpers.core.hookenv import (
45 is_leader as juju_is_leader, 46 is_leader as juju_is_leader,
46 status_set, 47 status_set,
47) 48)
49from charmhelpers.core.host import (
50 modulo_distribution,
51)
48from charmhelpers.core.decorators import ( 52from charmhelpers.core.decorators import (
49 retry_on_exception, 53 retry_on_exception,
50) 54)
@@ -361,3 +365,29 @@ def canonical_url(configs, vip_setting='vip'):
361 else: 365 else:
362 addr = unit_get('private-address') 366 addr = unit_get('private-address')
363 return '%s://%s' % (scheme, addr) 367 return '%s://%s' % (scheme, addr)
368
369
370def distributed_wait(modulo=None, wait=None, operation_name='operation'):
371 ''' Distribute operations by waiting based on modulo_distribution
372
373 If modulo and or wait are not set, check config_get for those values.
374
375 :param modulo: int The modulo number creates the group distribution
376 :param wait: int The constant time wait value
377 :param operation_name: string Operation name for status message
378 i.e. 'restart'
379 :side effect: Calls config_get()
380 :side effect: Calls log()
381 :side effect: Calls status_set()
382 :side effect: Calls time.sleep()
383 '''
384 if modulo is None:
385 modulo = config_get('modulo-nodes')
386 if wait is None:
387 wait = config_get('known-wait')
388 calculated_wait = modulo_distribution(modulo=modulo, wait=wait)
389 msg = "Waiting {} seconds for {} ...".format(calculated_wait,
390 operation_name)
391 log(msg, DEBUG)
392 status_set('maintenance', msg)
393 time.sleep(calculated_wait)
diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py
index d7e6deb..a871ce3 100644
--- a/hooks/charmhelpers/contrib/network/ip.py
+++ b/hooks/charmhelpers/contrib/network/ip.py
@@ -490,7 +490,7 @@ def get_host_ip(hostname, fallback=None):
490 if not ip_addr: 490 if not ip_addr:
491 try: 491 try:
492 ip_addr = socket.gethostbyname(hostname) 492 ip_addr = socket.gethostbyname(hostname)
493 except: 493 except Exception:
494 log("Failed to resolve hostname '%s'" % (hostname), 494 log("Failed to resolve hostname '%s'" % (hostname),
495 level=WARNING) 495 level=WARNING)
496 return fallback 496 return fallback
@@ -518,7 +518,7 @@ def get_hostname(address, fqdn=True):
518 if not result: 518 if not result:
519 try: 519 try:
520 result = socket.gethostbyaddr(address)[0] 520 result = socket.gethostbyaddr(address)[0]
521 except: 521 except Exception:
522 return None 522 return None
523 else: 523 else:
524 result = address 524 result = address
diff --git a/hooks/charmhelpers/contrib/openstack/alternatives.py b/hooks/charmhelpers/contrib/openstack/alternatives.py
index 1501641..547de09 100644
--- a/hooks/charmhelpers/contrib/openstack/alternatives.py
+++ b/hooks/charmhelpers/contrib/openstack/alternatives.py
@@ -29,3 +29,16 @@ def install_alternative(name, target, source, priority=50):
29 target, name, source, str(priority) 29 target, name, source, str(priority)
30 ] 30 ]
31 subprocess.check_call(cmd) 31 subprocess.check_call(cmd)
32
33
34def remove_alternative(name, source):
35 """Remove an installed alternative configuration file
36
37 :param name: string name of the alternative to remove
38 :param source: string full path to alternative to remove
39 """
40 cmd = [
41 'update-alternatives', '--remove',
42 name, source
43 ]
44 subprocess.check_call(cmd)
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
index 5c041d2..5afbbd8 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -13,6 +13,7 @@
13# limitations under the License. 13# limitations under the License.
14 14
15import logging 15import logging
16import os
16import re 17import re
17import sys 18import sys
18import six 19import six
@@ -185,7 +186,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
185 self.d.configure(service, config) 186 self.d.configure(service, config)
186 187
187 def _auto_wait_for_status(self, message=None, exclude_services=None, 188 def _auto_wait_for_status(self, message=None, exclude_services=None,
188 include_only=None, timeout=1800): 189 include_only=None, timeout=None):
189 """Wait for all units to have a specific extended status, except 190 """Wait for all units to have a specific extended status, except
190 for any defined as excluded. Unless specified via message, any 191 for any defined as excluded. Unless specified via message, any
191 status containing any case of 'ready' will be considered a match. 192 status containing any case of 'ready' will be considered a match.
@@ -215,7 +216,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
215 :param timeout: Maximum time in seconds to wait for status match 216 :param timeout: Maximum time in seconds to wait for status match
216 :returns: None. Raises if timeout is hit. 217 :returns: None. Raises if timeout is hit.
217 """ 218 """
218 self.log.info('Waiting for extended status on units...') 219 if not timeout:
220 timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 1800))
221 self.log.info('Waiting for extended status on units for {}s...'
222 ''.format(timeout))
219 223
220 all_services = self.d.services.keys() 224 all_services = self.d.services.keys()
221 225
@@ -250,7 +254,14 @@ class OpenStackAmuletDeployment(AmuletDeployment):
250 self.log.debug('Waiting up to {}s for extended status on services: ' 254 self.log.debug('Waiting up to {}s for extended status on services: '
251 '{}'.format(timeout, services)) 255 '{}'.format(timeout, services))
252 service_messages = {service: message for service in services} 256 service_messages = {service: message for service in services}
257
258 # Check for idleness
259 self.d.sentry.wait(timeout=timeout)
260 # Check for error states and bail early
261 self.d.sentry.wait_for_status(self.d.juju_env, services, timeout=timeout)
262 # Check for ready messages
253 self.d.sentry.wait_for_messages(service_messages, timeout=timeout) 263 self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
264
254 self.log.info('OK') 265 self.log.info('OK')
255 266
256 def _get_openstack_release(self): 267 def _get_openstack_release(self):
@@ -263,7 +274,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
263 (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty, 274 (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
264 self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton, 275 self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
265 self.yakkety_newton, self.xenial_ocata, self.zesty_ocata, 276 self.yakkety_newton, self.xenial_ocata, self.zesty_ocata,
266 self.xenial_pike, self.artful_pike) = range(11) 277 self.xenial_pike, self.artful_pike, self.xenial_queens,
278 self.bionic_queens,) = range(13)
267 279
268 releases = { 280 releases = {
269 ('trusty', None): self.trusty_icehouse, 281 ('trusty', None): self.trusty_icehouse,
@@ -274,9 +286,11 @@ class OpenStackAmuletDeployment(AmuletDeployment):
274 ('xenial', 'cloud:xenial-newton'): self.xenial_newton, 286 ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
275 ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata, 287 ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
276 ('xenial', 'cloud:xenial-pike'): self.xenial_pike, 288 ('xenial', 'cloud:xenial-pike'): self.xenial_pike,
289 ('xenial', 'cloud:xenial-queens'): self.xenial_queens,
277 ('yakkety', None): self.yakkety_newton, 290 ('yakkety', None): self.yakkety_newton,
278 ('zesty', None): self.zesty_ocata, 291 ('zesty', None): self.zesty_ocata,
279 ('artful', None): self.artful_pike, 292 ('artful', None): self.artful_pike,
293 ('bionic', None): self.bionic_queens,
280 } 294 }
281 return releases[(self.series, self.openstack)] 295 return releases[(self.series, self.openstack)]
282 296
@@ -291,6 +305,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
291 ('yakkety', 'newton'), 305 ('yakkety', 'newton'),
292 ('zesty', 'ocata'), 306 ('zesty', 'ocata'),
293 ('artful', 'pike'), 307 ('artful', 'pike'),
308 ('bionic', 'queens'),
294 ]) 309 ])
295 if self.openstack: 310 if self.openstack:
296 os_origin = self.openstack.split(':')[1] 311 os_origin = self.openstack.split(':')[1]
@@ -303,20 +318,27 @@ class OpenStackAmuletDeployment(AmuletDeployment):
303 test scenario, based on OpenStack release and whether ceph radosgw 318 test scenario, based on OpenStack release and whether ceph radosgw
304 is flagged as present or not.""" 319 is flagged as present or not."""
305 320
306 if self._get_openstack_release() >= self.trusty_kilo: 321 if self._get_openstack_release() == self.trusty_icehouse:
307 # Kilo or later 322 # Icehouse
308 pools = [ 323 pools = [
324 'data',
325 'metadata',
309 'rbd', 326 'rbd',
310 'cinder', 327 'cinder-ceph',
311 'glance' 328 'glance'
312 ] 329 ]
313 else: 330 elif (self.trusty_kilo <= self._get_openstack_release() <=
314 # Juno or earlier 331 self.zesty_ocata):
332 # Kilo through Ocata
315 pools = [ 333 pools = [
316 'data',
317 'metadata',
318 'rbd', 334 'rbd',
319 'cinder', 335 'cinder-ceph',
336 'glance'
337 ]
338 else:
339 # Pike and later
340 pools = [
341 'cinder-ceph',
320 'glance' 342 'glance'
321 ] 343 ]
322 344
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
index c8edbf6..b71b2b1 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
@@ -23,6 +23,7 @@ import urllib
23import urlparse 23import urlparse
24 24
25import cinderclient.v1.client as cinder_client 25import cinderclient.v1.client as cinder_client
26import cinderclient.v2.client as cinder_clientv2
26import glanceclient.v1.client as glance_client 27import glanceclient.v1.client as glance_client
27import heatclient.v1.client as heat_client 28import heatclient.v1.client as heat_client
28from keystoneclient.v2_0 import client as keystone_client 29from keystoneclient.v2_0 import client as keystone_client
@@ -42,7 +43,6 @@ import swiftclient
42from charmhelpers.contrib.amulet.utils import ( 43from charmhelpers.contrib.amulet.utils import (
43 AmuletUtils 44 AmuletUtils
44) 45)
45from charmhelpers.core.decorators import retry_on_exception
46from charmhelpers.core.host import CompareHostReleases 46from charmhelpers.core.host import CompareHostReleases
47 47
48DEBUG = logging.DEBUG 48DEBUG = logging.DEBUG
@@ -310,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils):
310 self.log.debug('Checking if tenant exists ({})...'.format(tenant)) 310 self.log.debug('Checking if tenant exists ({})...'.format(tenant))
311 return tenant in [t.name for t in keystone.tenants.list()] 311 return tenant in [t.name for t in keystone.tenants.list()]
312 312
313 @retry_on_exception(5, base_delay=10)
314 def keystone_wait_for_propagation(self, sentry_relation_pairs, 313 def keystone_wait_for_propagation(self, sentry_relation_pairs,
315 api_version): 314 api_version):
316 """Iterate over list of sentry and relation tuples and verify that 315 """Iterate over list of sentry and relation tuples and verify that
@@ -326,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils):
326 rel = sentry.relation('identity-service', 325 rel = sentry.relation('identity-service',
327 relation_name) 326 relation_name)
328 self.log.debug('keystone relation data: {}'.format(rel)) 327 self.log.debug('keystone relation data: {}'.format(rel))
329 if rel['api_version'] != str(api_version): 328 if rel.get('api_version') != str(api_version):
330 raise Exception("api_version not propagated through relation" 329 raise Exception("api_version not propagated through relation"
331 " data yet ('{}' != '{}')." 330 " data yet ('{}' != '{}')."
332 "".format(rel['api_version'], api_version)) 331 "".format(rel['api_version'], api_version))
@@ -348,15 +347,19 @@ class OpenStackAmuletUtils(AmuletUtils):
348 347
349 config = {'preferred-api-version': api_version} 348 config = {'preferred-api-version': api_version}
350 deployment.d.configure('keystone', config) 349 deployment.d.configure('keystone', config)
350 deployment._auto_wait_for_status()
351 self.keystone_wait_for_propagation(sentry_relation_pairs, api_version) 351 self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
352 352
353 def authenticate_cinder_admin(self, keystone_sentry, username, 353 def authenticate_cinder_admin(self, keystone_sentry, username,
354 password, tenant): 354 password, tenant, api_version=2):
355 """Authenticates admin user with cinder.""" 355 """Authenticates admin user with cinder."""
356 # NOTE(beisner): cinder python client doesn't accept tokens. 356 # NOTE(beisner): cinder python client doesn't accept tokens.
357 keystone_ip = keystone_sentry.info['public-address'] 357 keystone_ip = keystone_sentry.info['public-address']
358 ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8')) 358 ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
359 return cinder_client.Client(username, password, tenant, ept) 359 _clients = {
360 1: cinder_client.Client,
361 2: cinder_clientv2.Client}
362 return _clients[api_version](username, password, tenant, ept)
360 363
361 def authenticate_keystone(self, keystone_ip, username, password, 364 def authenticate_keystone(self, keystone_ip, username, password,
362 api_version=False, admin_port=False, 365 api_version=False, admin_port=False,
@@ -617,13 +620,25 @@ class OpenStackAmuletUtils(AmuletUtils):
617 self.log.debug('Keypair ({}) already exists, ' 620 self.log.debug('Keypair ({}) already exists, '
618 'using it.'.format(keypair_name)) 621 'using it.'.format(keypair_name))
619 return _keypair 622 return _keypair
620 except: 623 except Exception:
621 self.log.debug('Keypair ({}) does not exist, ' 624 self.log.debug('Keypair ({}) does not exist, '
622 'creating it.'.format(keypair_name)) 625 'creating it.'.format(keypair_name))
623 626
624 _keypair = nova.keypairs.create(name=keypair_name) 627 _keypair = nova.keypairs.create(name=keypair_name)
625 return _keypair 628 return _keypair
626 629
630 def _get_cinder_obj_name(self, cinder_object):
631 """Retrieve name of cinder object.
632
633 :param cinder_object: cinder snapshot or volume object
634 :returns: str cinder object name
635 """
636 # v1 objects store name in 'display_name' attr but v2+ use 'name'
637 try:
638 return cinder_object.display_name
639 except AttributeError:
640 return cinder_object.name
641
627 def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1, 642 def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
628 img_id=None, src_vol_id=None, snap_id=None): 643 img_id=None, src_vol_id=None, snap_id=None):
629 """Create cinder volume, optionally from a glance image, OR 644 """Create cinder volume, optionally from a glance image, OR
@@ -674,6 +689,13 @@ class OpenStackAmuletUtils(AmuletUtils):
674 source_volid=src_vol_id, 689 source_volid=src_vol_id,
675 snapshot_id=snap_id) 690 snapshot_id=snap_id)
676 vol_id = vol_new.id 691 vol_id = vol_new.id
692 except TypeError:
693 vol_new = cinder.volumes.create(name=vol_name,
694 imageRef=img_id,
695 size=vol_size,
696 source_volid=src_vol_id,
697 snapshot_id=snap_id)
698 vol_id = vol_new.id
677 except Exception as e: 699 except Exception as e:
678 msg = 'Failed to create volume: {}'.format(e) 700 msg = 'Failed to create volume: {}'.format(e)
679 amulet.raise_status(amulet.FAIL, msg=msg) 701 amulet.raise_status(amulet.FAIL, msg=msg)
@@ -688,7 +710,7 @@ class OpenStackAmuletUtils(AmuletUtils):
688 710
689 # Re-validate new volume 711 # Re-validate new volume
690 self.log.debug('Validating volume attributes...') 712 self.log.debug('Validating volume attributes...')
691 val_vol_name = cinder.volumes.get(vol_id).display_name 713 val_vol_name = self._get_cinder_obj_name(cinder.volumes.get(vol_id))
692 val_vol_boot = cinder.volumes.get(vol_id).bootable 714 val_vol_boot = cinder.volumes.get(vol_id).bootable
693 val_vol_stat = cinder.volumes.get(vol_id).status 715 val_vol_stat = cinder.volumes.get(vol_id).status
694 val_vol_size = cinder.volumes.get(vol_id).size 716 val_vol_size = cinder.volumes.get(vol_id).size
diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index f67f326..e6c0e9f 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -12,6 +12,7 @@
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14 14
15import collections
15import glob 16import glob
16import json 17import json
17import math 18import math
@@ -292,7 +293,7 @@ class PostgresqlDBContext(OSContextGenerator):
292def db_ssl(rdata, ctxt, ssl_dir): 293def db_ssl(rdata, ctxt, ssl_dir):
293 if 'ssl_ca' in rdata and ssl_dir: 294 if 'ssl_ca' in rdata and ssl_dir:
294 ca_path = os.path.join(ssl_dir, 'db-client.ca') 295 ca_path = os.path.join(ssl_dir, 'db-client.ca')
295 with open(ca_path, 'w') as fh: 296 with open(ca_path, 'wb') as fh:
296 fh.write(b64decode(rdata['ssl_ca'])) 297 fh.write(b64decode(rdata['ssl_ca']))
297 298
298 ctxt['database_ssl_ca'] = ca_path 299 ctxt['database_ssl_ca'] = ca_path
@@ -307,12 +308,12 @@ def db_ssl(rdata, ctxt, ssl_dir):
307 log("Waiting 1m for ssl client cert validity", level=INFO) 308 log("Waiting 1m for ssl client cert validity", level=INFO)
308 time.sleep(60) 309 time.sleep(60)
309 310
310 with open(cert_path, 'w') as fh: 311 with open(cert_path, 'wb') as fh:
311 fh.write(b64decode(rdata['ssl_cert'])) 312 fh.write(b64decode(rdata['ssl_cert']))
312 313
313 ctxt['database_ssl_cert'] = cert_path 314 ctxt['database_ssl_cert'] = cert_path
314 key_path = os.path.join(ssl_dir, 'db-client.key') 315 key_path = os.path.join(ssl_dir, 'db-client.key')
315 with open(key_path, 'w') as fh: 316 with open(key_path, 'wb') as fh:
316 fh.write(b64decode(rdata['ssl_key'])) 317 fh.write(b64decode(rdata['ssl_key']))
317 318
318 ctxt['database_ssl_key'] = key_path 319 ctxt['database_ssl_key'] = key_path
@@ -458,7 +459,7 @@ class AMQPContext(OSContextGenerator):
458 459
459 ca_path = os.path.join( 460 ca_path = os.path.join(
460 self.ssl_dir, 'rabbit-client-ca.pem') 461 self.ssl_dir, 'rabbit-client-ca.pem')
461 with open(ca_path, 'w') as fh: 462 with open(ca_path, 'wb') as fh:
462 fh.write(b64decode(ctxt['rabbit_ssl_ca'])) 463 fh.write(b64decode(ctxt['rabbit_ssl_ca']))
463 ctxt['rabbit_ssl_ca'] = ca_path 464 ctxt['rabbit_ssl_ca'] = ca_path
464 465
@@ -578,11 +579,14 @@ class HAProxyContext(OSContextGenerator):
578 laddr = get_address_in_network(config(cfg_opt)) 579 laddr = get_address_in_network(config(cfg_opt))
579 if laddr: 580 if laddr:
580 netmask = get_netmask_for_address(laddr) 581 netmask = get_netmask_for_address(laddr)
581 cluster_hosts[laddr] = {'network': "{}/{}".format(laddr, 582 cluster_hosts[laddr] = {
582 netmask), 583 'network': "{}/{}".format(laddr,
583 'backends': {l_unit: laddr}} 584 netmask),
585 'backends': collections.OrderedDict([(l_unit,
586 laddr)])
587 }
584 for rid in relation_ids('cluster'): 588 for rid in relation_ids('cluster'):
585 for unit in related_units(rid): 589 for unit in sorted(related_units(rid)):
586 _laddr = relation_get('{}-address'.format(addr_type), 590 _laddr = relation_get('{}-address'.format(addr_type),
587 rid=rid, unit=unit) 591 rid=rid, unit=unit)
588 if _laddr: 592 if _laddr:
@@ -594,10 +598,13 @@ class HAProxyContext(OSContextGenerator):
594 # match in the frontend 598 # match in the frontend
595 cluster_hosts[addr] = {} 599 cluster_hosts[addr] = {}
596 netmask = get_netmask_for_address(addr) 600 netmask = get_netmask_for_address(addr)
597 cluster_hosts[addr] = {'network': "{}/{}".format(addr, netmask), 601 cluster_hosts[addr] = {
598 'backends': {l_unit: addr}} 602 'network': "{}/{}".format(addr, netmask),
603 'backends': collections.OrderedDict([(l_unit,
604 addr)])
605 }
599 for rid in relation_ids('cluster'): 606 for rid in relation_ids('cluster'):
600 for unit in related_units(rid): 607 for unit in sorted(related_units(rid)):
601 _laddr = relation_get('private-address', 608 _laddr = relation_get('private-address',
602 rid=rid, unit=unit) 609 rid=rid, unit=unit)
603 if _laddr: 610 if _laddr:
@@ -628,6 +635,8 @@ class HAProxyContext(OSContextGenerator):
628 ctxt['local_host'] = '127.0.0.1' 635 ctxt['local_host'] = '127.0.0.1'
629 ctxt['haproxy_host'] = '0.0.0.0' 636 ctxt['haproxy_host'] = '0.0.0.0'
630 637
638 ctxt['ipv6_enabled'] = not is_ipv6_disabled()
639
631 ctxt['stat_port'] = '8888' 640 ctxt['stat_port'] = '8888'
632 641
633 db = kv() 642 db = kv()
@@ -802,8 +811,9 @@ class ApacheSSLContext(OSContextGenerator):
802 else: 811 else:
803 # Expect cert/key provided in config (currently assumed that ca 812 # Expect cert/key provided in config (currently assumed that ca
804 # uses ip for cn) 813 # uses ip for cn)
805 cn = resolve_address(endpoint_type=INTERNAL) 814 for net_type in (INTERNAL, ADMIN, PUBLIC):
806 self.configure_cert(cn) 815 cn = resolve_address(endpoint_type=net_type)
816 self.configure_cert(cn)
807 817
808 addresses = self.get_network_addresses() 818 addresses = self.get_network_addresses()
809 for address, endpoint in addresses: 819 for address, endpoint in addresses:
@@ -843,15 +853,6 @@ class NeutronContext(OSContextGenerator):
843 for pkgs in self.packages: 853 for pkgs in self.packages:
844 ensure_packages(pkgs) 854 ensure_packages(pkgs)
845 855
846 def _save_flag_file(self):
847 if self.network_manager == 'quantum':
848 _file = '/etc/nova/quantum_plugin.conf'
849 else:
850 _file = '/etc/nova/neutron_plugin.conf'
851
852 with open(_file, 'wb') as out:
853 out.write(self.plugin + '\n')
854
855 def ovs_ctxt(self): 856 def ovs_ctxt(self):
856 driver = neutron_plugin_attribute(self.plugin, 'driver', 857 driver = neutron_plugin_attribute(self.plugin, 'driver',
857 self.network_manager) 858 self.network_manager)
@@ -996,7 +997,6 @@ class NeutronContext(OSContextGenerator):
996 flags = config_flags_parser(alchemy_flags) 997 flags = config_flags_parser(alchemy_flags)
997 ctxt['neutron_alchemy_flags'] = flags 998 ctxt['neutron_alchemy_flags'] = flags
998 999
999 self._save_flag_file()
1000 return ctxt 1000 return ctxt
1001 1001
1002 1002
@@ -1176,7 +1176,7 @@ class SubordinateConfigContext(OSContextGenerator):
1176 if sub_config and sub_config != '': 1176 if sub_config and sub_config != '':
1177 try: 1177 try:
1178 sub_config = json.loads(sub_config) 1178 sub_config = json.loads(sub_config)
1179 except: 1179 except Exception:
1180 log('Could not parse JSON from ' 1180 log('Could not parse JSON from '
1181 'subordinate_configuration setting from %s' 1181 'subordinate_configuration setting from %s'
1182 % rid, level=ERROR) 1182 % rid, level=ERROR)
diff --git a/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
index 0df0717..7aab129 100755
--- a/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
+++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
@@ -9,7 +9,7 @@
9CRITICAL=0 9CRITICAL=0
10NOTACTIVE='' 10NOTACTIVE=''
11LOGFILE=/var/log/nagios/check_haproxy.log 11LOGFILE=/var/log/nagios/check_haproxy.log
12AUTH=$(grep -r "stats auth" /etc/haproxy | awk 'NR=1{print $4}') 12AUTH=$(grep -r "stats auth" /etc/haproxy/haproxy.cfg | awk 'NR=1{print $4}')
13 13
14typeset -i N_INSTANCES=0 14typeset -i N_INSTANCES=0
15for appserver in $(awk '/^\s+server/{print $2}' /etc/haproxy/haproxy.cfg) 15for appserver in $(awk '/^\s+server/{print $2}' /etc/haproxy/haproxy.cfg)
diff --git a/hooks/charmhelpers/contrib/openstack/ha/utils.py b/hooks/charmhelpers/contrib/openstack/ha/utils.py
index 254a90e..9a4d79c 100644
--- a/hooks/charmhelpers/contrib/openstack/ha/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/ha/utils.py
@@ -82,15 +82,18 @@ def update_dns_ha_resource_params(resources, resource_params,
82 continue 82 continue
83 m = re.search('os-(.+?)-hostname', setting) 83 m = re.search('os-(.+?)-hostname', setting)
84 if m: 84 if m:
85 networkspace = m.group(1) 85 endpoint_type = m.group(1)
86 # resolve_address's ADDRESS_MAP uses 'int' not 'internal'
87 if endpoint_type == 'internal':
88 endpoint_type = 'int'
86 else: 89 else:
87 msg = ('Unexpected DNS hostname setting: {}. ' 90 msg = ('Unexpected DNS hostname setting: {}. '
88 'Cannot determine network space name' 91 'Cannot determine endpoint_type name'
89 ''.format(setting)) 92 ''.format(setting))
90 status_set('blocked', msg) 93 status_set('blocked', msg)
91 raise DNSHAException(msg) 94 raise DNSHAException(msg)
92 95
93 hostname_key = 'res_{}_{}_hostname'.format(charm_name(), networkspace) 96 hostname_key = 'res_{}_{}_hostname'.format(charm_name(), endpoint_type)
94 if hostname_key in hostname_group: 97 if hostname_key in hostname_group:
95 log('DNS HA: Resource {}: {} already exists in ' 98 log('DNS HA: Resource {}: {} already exists in '
96 'hostname group - skipping'.format(hostname_key, hostname), 99 'hostname group - skipping'.format(hostname_key, hostname),
@@ -101,7 +104,7 @@ def update_dns_ha_resource_params(resources, resource_params,
101 resources[hostname_key] = crm_ocf 104 resources[hostname_key] = crm_ocf
102 resource_params[hostname_key] = ( 105 resource_params[hostname_key] = (
103 'params fqdn="{}" ip_address="{}" ' 106 'params fqdn="{}" ip_address="{}" '
104 ''.format(hostname, resolve_address(endpoint_type=networkspace, 107 ''.format(hostname, resolve_address(endpoint_type=endpoint_type,
105 override=False))) 108 override=False)))
106 109
107 if len(hostname_group) >= 1: 110 if len(hostname_group) >= 1:
diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py
index 37fa0eb..0f847f5 100644
--- a/hooks/charmhelpers/contrib/openstack/neutron.py
+++ b/hooks/charmhelpers/contrib/openstack/neutron.py
@@ -59,18 +59,13 @@ def determine_dkms_package():
59 59
60 60
61def quantum_plugins(): 61def quantum_plugins():
62 from charmhelpers.contrib.openstack import context
63 return { 62 return {
64 'ovs': { 63 'ovs': {
65 'config': '/etc/quantum/plugins/openvswitch/' 64 'config': '/etc/quantum/plugins/openvswitch/'
66 'ovs_quantum_plugin.ini', 65 'ovs_quantum_plugin.ini',
67 'driver': 'quantum.plugins.openvswitch.ovs_quantum_plugin.' 66 'driver': 'quantum.plugins.openvswitch.ovs_quantum_plugin.'
68 'OVSQuantumPluginV2', 67 'OVSQuantumPluginV2',
69 'contexts': [ 68 'contexts': [],
70 context.SharedDBContext(user=config('neutron-database-user'),
71 database=config('neutron-database'),
72 relation_prefix='neutron',
73 ssl_dir=QUANTUM_CONF_DIR)],
74 'services': ['quantum-plugin-openvswitch-agent'], 69 'services': ['quantum-plugin-openvswitch-agent'],
75 'packages': [determine_dkms_package(), 70 'packages': [determine_dkms_package(),
76 ['quantum-plugin-openvswitch-agent']], 71 ['quantum-plugin-openvswitch-agent']],
@@ -82,11 +77,7 @@ def quantum_plugins():
82 'config': '/etc/quantum/plugins/nicira/nvp.ini', 77 'config': '/etc/quantum/plugins/nicira/nvp.ini',
83 'driver': 'quantum.plugins.nicira.nicira_nvp_plugin.' 78 'driver': 'quantum.plugins.nicira.nicira_nvp_plugin.'
84 'QuantumPlugin.NvpPluginV2', 79 'QuantumPlugin.NvpPluginV2',
85 'contexts': [ 80 'contexts': [],
86 context.SharedDBContext(user=config('neutron-database-user'),
87 database=config('neutron-database'),
88 relation_prefix='neutron',
89 ssl_dir=QUANTUM_CONF_DIR)],
90 'services': [], 81 'services': [],
91 'packages': [], 82 'packages': [],
92 'server_packages': ['quantum-server', 83 'server_packages': ['quantum-server',
@@ -100,7 +91,6 @@ NEUTRON_CONF_DIR = '/etc/neutron'
100 91
101 92
102def neutron_plugins(): 93def neutron_plugins():
103 from charmhelpers.contrib.openstack import context
104 release = os_release('nova-common') 94 release = os_release('nova-common')
105 plugins = { 95 plugins = {
106 'ovs': { 96 'ovs': {
@@ -108,11 +98,7 @@ def neutron_plugins():
108 'ovs_neutron_plugin.ini', 98 'ovs_neutron_plugin.ini',
109 'driver': 'neutron.plugins.openvswitch.ovs_neutron_plugin.' 99 'driver': 'neutron.plugins.openvswitch.ovs_neutron_plugin.'
110 'OVSNeutronPluginV2', 100 'OVSNeutronPluginV2',
111 'contexts': [ 101 'contexts': [],
112 context.SharedDBContext(user=config('neutron-database-user'),
113 database=config('neutron-database'),
114 relation_prefix='neutron',
115 ssl_dir=NEUTRON_CONF_DIR)],
116 'services': ['neutron-plugin-openvswitch-agent'], 102 'services': ['neutron-plugin-openvswitch-agent'],
117 'packages': [determine_dkms_package(), 103 'packages': [determine_dkms_package(),
118 ['neutron-plugin-openvswitch-agent']], 104 ['neutron-plugin-openvswitch-agent']],
@@ -124,11 +110,7 @@ def neutron_plugins():
124 'config': '/etc/neutron/plugins/nicira/nvp.ini', 110 'config': '/etc/neutron/plugins/nicira/nvp.ini',
125 'driver': 'neutron.plugins.nicira.nicira_nvp_plugin.' 111 'driver': 'neutron.plugins.nicira.nicira_nvp_plugin.'
126 'NeutronPlugin.NvpPluginV2', 112 'NeutronPlugin.NvpPluginV2',
127 'contexts': [ 113 'contexts': [],
128 context.SharedDBContext(user=config('neutron-database-user'),
129 database=config('neutron-database'),
130 relation_prefix='neutron',
131 ssl_dir=NEUTRON_CONF_DIR)],
132 'services': [], 114 'services': [],
133 'packages': [], 115 'packages': [],
134 'server_packages': ['neutron-server', 116 'server_packages': ['neutron-server',
@@ -138,11 +120,7 @@ def neutron_plugins():
138 'nsx': { 120 'nsx': {
139 'config': '/etc/neutron/plugins/vmware/nsx.ini', 121 'config': '/etc/neutron/plugins/vmware/nsx.ini',
140 'driver': 'vmware', 122 'driver': 'vmware',
141 'contexts': [ 123 'contexts': [],
142 context.SharedDBContext(user=config('neutron-database-user'),
143 database=config('neutron-database'),
144 relation_prefix='neutron',
145 ssl_dir=NEUTRON_CONF_DIR)],
146 'services': [], 124 'services': [],
147 'packages': [], 125 'packages': [],
148 'server_packages': ['neutron-server', 126 'server_packages': ['neutron-server',
@@ -152,11 +130,7 @@ def neutron_plugins():
152 'n1kv': { 130 'n1kv': {
153 'config': '/etc/neutron/plugins/cisco/cisco_plugins.ini', 131 'config': '/etc/neutron/plugins/cisco/cisco_plugins.ini',
154 'driver': 'neutron.plugins.cisco.network_plugin.PluginV2', 132 'driver': 'neutron.plugins.cisco.network_plugin.PluginV2',
155 'contexts': [ 133 'contexts': [],
156 context.SharedDBContext(user=config('neutron-database-user'),
157 database=config('neutron-database'),
158 relation_prefix='neutron',
159 ssl_dir=NEUTRON_CONF_DIR)],
160 'services': [], 134 'services': [],
161 'packages': [determine_dkms_package(), 135 'packages': [determine_dkms_package(),
162 ['neutron-plugin-cisco']], 136 ['neutron-plugin-cisco']],
@@ -167,11 +141,7 @@ def neutron_plugins():
167 'Calico': { 141 'Calico': {
168 'config': '/etc/neutron/plugins/ml2/ml2_conf.ini', 142 'config': '/etc/neutron/plugins/ml2/ml2_conf.ini',
169 'driver': 'neutron.plugins.ml2.plugin.Ml2Plugin', 143 'driver': 'neutron.plugins.ml2.plugin.Ml2Plugin',
170 'contexts': [ 144 'contexts': [],
171 context.SharedDBContext(user=config('neutron-database-user'),
172 database=config('neutron-database'),
173 relation_prefix='neutron',
174 ssl_dir=NEUTRON_CONF_DIR)],
175 'services': ['calico-felix', 145 'services': ['calico-felix',
176 'bird', 146 'bird',
177 'neutron-dhcp-agent', 147 'neutron-dhcp-agent',
@@ -189,11 +159,7 @@ def neutron_plugins():
189 'vsp': { 159 'vsp': {
190 'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini', 160 'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
191 'driver': 'neutron.plugins.nuage.plugin.NuagePlugin', 161 'driver': 'neutron.plugins.nuage.plugin.NuagePlugin',
192 'contexts': [ 162 'contexts': [],
193 context.SharedDBContext(user=config('neutron-database-user'),
194 database=config('neutron-database'),
195 relation_prefix='neutron',
196 ssl_dir=NEUTRON_CONF_DIR)],
197 'services': [], 163 'services': [],
198 'packages': [], 164 'packages': [],
199 'server_packages': ['neutron-server', 'neutron-plugin-nuage'], 165 'server_packages': ['neutron-server', 'neutron-plugin-nuage'],
@@ -203,10 +169,7 @@ def neutron_plugins():
203 'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini', 169 'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini',
204 'driver': ('neutron.plugins.plumgrid.plumgrid_plugin' 170 'driver': ('neutron.plugins.plumgrid.plumgrid_plugin'
205 '.plumgrid_plugin.NeutronPluginPLUMgridV2'), 171 '.plumgrid_plugin.NeutronPluginPLUMgridV2'),
206 'contexts': [ 172 'contexts': [],
207 context.SharedDBContext(user=config('database-user'),
208 database=config('database'),
209 ssl_dir=NEUTRON_CONF_DIR)],
210 'services': [], 173 'services': [],
211 'packages': ['plumgrid-lxc', 174 'packages': ['plumgrid-lxc',
212 'iovisor-dkms'], 175 'iovisor-dkms'],
@@ -217,11 +180,7 @@ def neutron_plugins():
217 'midonet': { 180 'midonet': {
218 'config': '/etc/neutron/plugins/midonet/midonet.ini', 181 'config': '/etc/neutron/plugins/midonet/midonet.ini',
219 'driver': 'midonet.neutron.plugin.MidonetPluginV2', 182 'driver': 'midonet.neutron.plugin.MidonetPluginV2',
220 'contexts': [ 183 'contexts': [],
221 context.SharedDBContext(user=config('neutron-database-user'),
222 database=config('neutron-database'),
223 relation_prefix='neutron',
224 ssl_dir=NEUTRON_CONF_DIR)],
225 'services': [], 184 'services': [],
226 'packages': [determine_dkms_package()], 185 'packages': [determine_dkms_package()],
227 'server_packages': ['neutron-server', 186 'server_packages': ['neutron-server',
diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf
index ed5c4f1..a11ce8a 100644
--- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf
+++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf
@@ -18,7 +18,7 @@ rbd default features = {{ rbd_features }}
18 18
19[client] 19[client]
20{% if rbd_client_cache_settings -%} 20{% if rbd_client_cache_settings -%}
21{% for key, value in rbd_client_cache_settings.iteritems() -%} 21{% for key, value in rbd_client_cache_settings.items() -%}
22{{ key }} = {{ value }} 22{{ key }} = {{ value }}
23{% endfor -%} 23{% endfor -%}
24{%- endif %} \ No newline at end of file 24{%- endif %}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg
index 2e66045..ebc8a68 100644
--- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg
+++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg
@@ -48,7 +48,9 @@ listen stats
48{% for service, ports in service_ports.items() -%} 48{% for service, ports in service_ports.items() -%}
49frontend tcp-in_{{ service }} 49frontend tcp-in_{{ service }}
50 bind *:{{ ports[0] }} 50 bind *:{{ ports[0] }}
51 {% if ipv6_enabled -%}
51 bind :::{{ ports[0] }} 52 bind :::{{ ports[0] }}
53 {% endif -%}
52 {% for frontend in frontends -%} 54 {% for frontend in frontends -%}
53 acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }} 55 acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }}
54 use_backend {{ service }}_{{ frontend }} if net_{{ frontend }} 56 use_backend {{ service }}_{{ frontend }} if net_{{ frontend }}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-oslo-cache b/hooks/charmhelpers/contrib/openstack/templates/section-oslo-cache
new file mode 100644
index 0000000..e056a32
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/templates/section-oslo-cache
@@ -0,0 +1,6 @@
1[cache]
2{% if memcache_url %}
3enabled = true
4backend = oslo_cache.memcache_pool
5memcache_servers = {{ memcache_url }}
6{% endif %}
diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py
index d8c1fc7..77490e4 100644
--- a/hooks/charmhelpers/contrib/openstack/templating.py
+++ b/hooks/charmhelpers/contrib/openstack/templating.py
@@ -272,6 +272,8 @@ class OSConfigRenderer(object):
272 raise OSConfigException 272 raise OSConfigException
273 273
274 _out = self.render(config_file) 274 _out = self.render(config_file)
275 if six.PY3:
276 _out = _out.encode('UTF-8')
275 277
276 with open(config_file, 'wb') as out: 278 with open(config_file, 'wb') as out:
277 out.write(_out) 279 out.write(_out)
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 837a167..8a541d4 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -95,7 +95,7 @@ from charmhelpers.fetch import (
95from charmhelpers.fetch.snap import ( 95from charmhelpers.fetch.snap import (
96 snap_install, 96 snap_install,
97 snap_refresh, 97 snap_refresh,
98 SNAP_CHANNELS, 98 valid_snap_channel,
99) 99)
100 100
101from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk 101from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
@@ -140,6 +140,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
140 ('yakkety', 'newton'), 140 ('yakkety', 'newton'),
141 ('zesty', 'ocata'), 141 ('zesty', 'ocata'),
142 ('artful', 'pike'), 142 ('artful', 'pike'),
143 ('bionic', 'queens'),
143]) 144])
144 145
145 146
@@ -157,6 +158,7 @@ OPENSTACK_CODENAMES = OrderedDict([
157 ('2016.2', 'newton'), 158 ('2016.2', 'newton'),
158 ('2017.1', 'ocata'), 159 ('2017.1', 'ocata'),
159 ('2017.2', 'pike'), 160 ('2017.2', 'pike'),
161 ('2018.1', 'queens'),
160]) 162])
161 163
162# The ugly duckling - must list releases oldest to newest 164# The ugly duckling - must list releases oldest to newest
@@ -187,6 +189,8 @@ SWIFT_CODENAMES = OrderedDict([
187 ['2.11.0', '2.12.0', '2.13.0']), 189 ['2.11.0', '2.12.0', '2.13.0']),
188 ('pike', 190 ('pike',
189 ['2.13.0', '2.15.0']), 191 ['2.13.0', '2.15.0']),
192 ('queens',
193 ['2.16.0']),
190]) 194])
191 195
192# >= Liberty version->codename mapping 196# >= Liberty version->codename mapping
@@ -412,6 +416,8 @@ def get_os_codename_package(package, fatal=True):
412 cmd = ['snap', 'list', package] 416 cmd = ['snap', 'list', package]
413 try: 417 try:
414 out = subprocess.check_output(cmd) 418 out = subprocess.check_output(cmd)
419 if six.PY3:
420 out = out.decode('UTF-8')
415 except subprocess.CalledProcessError as e: 421 except subprocess.CalledProcessError as e:
416 return None 422 return None
417 lines = out.split('\n') 423 lines = out.split('\n')
@@ -426,7 +432,7 @@ def get_os_codename_package(package, fatal=True):
426 432
427 try: 433 try:
428 pkg = cache[package] 434 pkg = cache[package]
429 except: 435 except Exception:
430 if not fatal: 436 if not fatal:
431 return None 437 return None
432 # the package is unknown to the current apt cache. 438 # the package is unknown to the current apt cache.
@@ -579,6 +585,9 @@ def configure_installation_source(source_plus_key):
579 Note that the behaviour on error is to log the error to the juju log and 585 Note that the behaviour on error is to log the error to the juju log and
580 then call sys.exit(1). 586 then call sys.exit(1).
581 """ 587 """
588 if source_plus_key.startswith('snap'):
589 # Do nothing for snap installs
590 return
582 # extract the key if there is one, denoted by a '|' in the rel 591 # extract the key if there is one, denoted by a '|' in the rel
583 source, key = get_source_and_pgp_key(source_plus_key) 592 source, key = get_source_and_pgp_key(source_plus_key)
584 593
@@ -615,7 +624,7 @@ def save_script_rc(script_path="scripts/scriptrc", **env_vars):
615 juju_rc_path = "%s/%s" % (charm_dir(), script_path) 624 juju_rc_path = "%s/%s" % (charm_dir(), script_path)
616 if not os.path.exists(os.path.dirname(juju_rc_path)): 625 if not os.path.exists(os.path.dirname(juju_rc_path)):
617 os.mkdir(os.path.dirname(juju_rc_path)) 626 os.mkdir(os.path.dirname(juju_rc_path))
618 with open(juju_rc_path, 'wb') as rc_script: 627 with open(juju_rc_path, 'wt') as rc_script:
619 rc_script.write( 628 rc_script.write(
620 "#!/bin/bash\n") 629 "#!/bin/bash\n")
621 [rc_script.write('export %s=%s\n' % (u, p)) 630 [rc_script.write('export %s=%s\n' % (u, p))
@@ -794,7 +803,7 @@ def git_default_repos(projects_yaml):
794 service = service_name() 803 service = service_name()
795 core_project = service 804 core_project = service
796 805
797 for default, branch in GIT_DEFAULT_BRANCHES.iteritems(): 806 for default, branch in six.iteritems(GIT_DEFAULT_BRANCHES):
798 if projects_yaml == default: 807 if projects_yaml == default:
799 808
800 # add the requirements repo first 809 # add the requirements repo first
@@ -1615,7 +1624,7 @@ def do_action_openstack_upgrade(package, upgrade_callback, configs):
1615 upgrade_callback(configs=configs) 1624 upgrade_callback(configs=configs)
1616 action_set({'outcome': 'success, upgrade completed.'}) 1625 action_set({'outcome': 'success, upgrade completed.'})
1617 ret = True 1626 ret = True
1618 except: 1627 except Exception:
1619 action_set({'outcome': 'upgrade failed, see traceback.'}) 1628 action_set({'outcome': 'upgrade failed, see traceback.'})
1620 action_set({'traceback': traceback.format_exc()}) 1629 action_set({'traceback': traceback.format_exc()})
1621 action_fail('do_openstack_upgrade resulted in an ' 1630 action_fail('do_openstack_upgrade resulted in an '
@@ -1720,7 +1729,7 @@ def is_unit_paused_set():
1720 kv = t[0] 1729 kv = t[0]
1721 # transform something truth-y into a Boolean. 1730 # transform something truth-y into a Boolean.
1722 return not(not(kv.get('unit-paused'))) 1731 return not(not(kv.get('unit-paused')))
1723 except: 1732 except Exception:
1724 return False 1733 return False
1725 1734
1726 1735
@@ -2048,7 +2057,7 @@ def update_json_file(filename, items):
2048def snap_install_requested(): 2057def snap_install_requested():
2049 """ Determine if installing from snaps 2058 """ Determine if installing from snaps
2050 2059
2051 If openstack-origin is of the form snap:channel-series-release 2060 If openstack-origin is of the form snap:track/channel[/branch]
2052 and channel is in SNAPS_CHANNELS return True. 2061 and channel is in SNAPS_CHANNELS return True.
2053 """ 2062 """
2054 origin = config('openstack-origin') or "" 2063 origin = config('openstack-origin') or ""
@@ -2056,10 +2065,12 @@ def snap_install_requested():
2056 return False 2065 return False
2057 2066
2058 _src = origin[5:] 2067 _src = origin[5:]
2059 channel, series, release = _src.split('-') 2068 if '/' in _src:
2060 if channel.lower() in SNAP_CHANNELS: 2069 channel = _src.split('/')[1]
2061 return True 2070 else:
2062 return False 2071 # Handle snap:track with no channel
2072 channel = 'stable'
2073 return valid_snap_channel(channel)
2063 2074
2064 2075
2065def get_snaps_install_info_from_origin(snaps, src, mode='classic'): 2076def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
@@ -2067,7 +2078,7 @@ def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
2067 2078
2068 @param snaps: List of snaps 2079 @param snaps: List of snaps
2069 @param src: String of openstack-origin or source of the form 2080 @param src: String of openstack-origin or source of the form
2070 snap:channel-series-track 2081 snap:track/channel
2071 @param mode: String classic, devmode or jailmode 2082 @param mode: String classic, devmode or jailmode
2072 @returns: Dictionary of snaps with channels and modes 2083 @returns: Dictionary of snaps with channels and modes
2073 """ 2084 """
@@ -2077,8 +2088,7 @@ def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
2077 return {} 2088 return {}
2078 2089
2079 _src = src[5:] 2090 _src = src[5:]
2080 _channel, _series, _release = _src.split('-') 2091 channel = '--channel={}'.format(_src)
2081 channel = '--channel={}/{}'.format(_release, _channel)
2082 2092
2083 return {snap: {'channel': channel, 'mode': mode} 2093 return {snap: {'channel': channel, 'mode': mode}
2084 for snap in snaps} 2094 for snap in snaps}
@@ -2090,8 +2100,8 @@ def install_os_snaps(snaps, refresh=False):
2090 @param snaps: Dictionary of snaps with channels and modes of the form: 2100 @param snaps: Dictionary of snaps with channels and modes of the form:
2091 {'snap_name': {'channel': 'snap_channel', 2101 {'snap_name': {'channel': 'snap_channel',
2092 'mode': 'snap_mode'}} 2102 'mode': 'snap_mode'}}
2093 Where channel a snapstore channel and mode is --classic, --devmode or 2103 Where channel is a snapstore channel and mode is --classic, --devmode
2094 --jailmode. 2104 or --jailmode.
2095 @param post_snap_install: Callback function to run after snaps have been 2105 @param post_snap_install: Callback function to run after snaps have been
2096 installed 2106 installed
2097 """ 2107 """
diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py
index e5a01b1..3923161 100644
--- a/hooks/charmhelpers/contrib/storage/linux/ceph.py
+++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py
@@ -370,9 +370,10 @@ def get_mon_map(service):
370 Also raises CalledProcessError if our ceph command fails 370 Also raises CalledProcessError if our ceph command fails
371 """ 371 """
372 try: 372 try:
373 mon_status = check_output( 373 mon_status = check_output(['ceph', '--id', service,
374 ['ceph', '--id', service, 374 'mon_status', '--format=json'])
375 'mon_status', '--format=json']) 375 if six.PY3:
376 mon_status = mon_status.decode('UTF-8')
376 try: 377 try:
377 return json.loads(mon_status) 378 return json.loads(mon_status)
378 except ValueError as v: 379 except ValueError as v:
@@ -457,7 +458,7 @@ def monitor_key_get(service, key):
457 try: 458 try:
458 output = check_output( 459 output = check_output(
459 ['ceph', '--id', service, 460 ['ceph', '--id', service,
460 'config-key', 'get', str(key)]) 461 'config-key', 'get', str(key)]).decode('UTF-8')
461 return output 462 return output
462 except CalledProcessError as e: 463 except CalledProcessError as e:
463 log("Monitor config-key get failed with message: {}".format( 464 log("Monitor config-key get failed with message: {}".format(
@@ -500,6 +501,8 @@ def get_erasure_profile(service, name):
500 out = check_output(['ceph', '--id', service, 501 out = check_output(['ceph', '--id', service,
501 'osd', 'erasure-code-profile', 'get', 502 'osd', 'erasure-code-profile', 'get',
502 name, '--format=json']) 503 name, '--format=json'])
504 if six.PY3:
505 out = out.decode('UTF-8')
503 return json.loads(out) 506 return json.loads(out)
504 except (CalledProcessError, OSError, ValueError): 507 except (CalledProcessError, OSError, ValueError):
505 return None 508 return None
@@ -686,7 +689,10 @@ def get_cache_mode(service, pool_name):
686 """ 689 """
687 validator(value=service, valid_type=six.string_types) 690 validator(value=service, valid_type=six.string_types)
688 validator(value=pool_name, valid_type=six.string_types) 691 validator(value=pool_name, valid_type=six.string_types)
689 out = check_output(['ceph', '--id', service, 'osd', 'dump', '--format=json']) 692 out = check_output(['ceph', '--id', service,
693 'osd', 'dump', '--format=json'])
694 if six.PY3:
695 out = out.decode('UTF-8')
690 try: 696 try:
691 osd_json = json.loads(out) 697 osd_json = json.loads(out)
692 for pool in osd_json['pools']: 698 for pool in osd_json['pools']:
@@ -700,8 +706,9 @@ def get_cache_mode(service, pool_name):
700def pool_exists(service, name): 706def pool_exists(service, name):
701 """Check to see if a RADOS pool already exists.""" 707 """Check to see if a RADOS pool already exists."""
702 try: 708 try:
703 out = check_output(['rados', '--id', service, 709 out = check_output(['rados', '--id', service, 'lspools'])
704 'lspools']).decode('UTF-8') 710 if six.PY3:
711 out = out.decode('UTF-8')
705 except CalledProcessError: 712 except CalledProcessError:
706 return False 713 return False
707 714
@@ -714,9 +721,12 @@ def get_osds(service):
714 """ 721 """
715 version = ceph_version() 722 version = ceph_version()
716 if version and version >= '0.56': 723 if version and version >= '0.56':
717 return json.loads(check_output(['ceph', '--id', service, 724 out = check_output(['ceph', '--id', service,
718 'osd', 'ls', 725 'osd', 'ls',
719 '--format=json']).decode('UTF-8')) 726 '--format=json'])
727 if six.PY3:
728 out = out.decode('UTF-8')
729 return json.loads(out)
720 730
721 return None 731 return None
722 732
@@ -734,7 +744,9 @@ def rbd_exists(service, pool, rbd_img):
734 """Check to see if a RADOS block device exists.""" 744 """Check to see if a RADOS block device exists."""
735 try: 745 try:
736 out = check_output(['rbd', 'list', '--id', 746 out = check_output(['rbd', 'list', '--id',
737 service, '--pool', pool]).decode('UTF-8') 747 service, '--pool', pool])
748 if six.PY3:
749 out = out.decode('UTF-8')
738 except CalledProcessError: 750 except CalledProcessError:
739 return False 751 return False
740 752
@@ -859,7 +871,9 @@ def configure(service, key, auth, use_syslog):
859def image_mapped(name): 871def image_mapped(name):
860 """Determine whether a RADOS block device is mapped locally.""" 872 """Determine whether a RADOS block device is mapped locally."""
861 try: 873 try:
862 out = check_output(['rbd', 'showmapped']).decode('UTF-8') 874 out = check_output(['rbd', 'showmapped'])
875 if six.PY3:
876 out = out.decode('UTF-8')
863 except CalledProcessError: 877 except CalledProcessError:
864 return False 878 return False
865 879
@@ -1018,7 +1032,9 @@ def ceph_version():
1018 """Retrieve the local version of ceph.""" 1032 """Retrieve the local version of ceph."""
1019 if os.path.exists('/usr/bin/ceph'): 1033 if os.path.exists('/usr/bin/ceph'):
1020 cmd = ['ceph', '-v'] 1034 cmd = ['ceph', '-v']
1021 output = check_output(cmd).decode('US-ASCII') 1035 output = check_output(cmd)
1036 if six.PY3:
1037 output = output.decode('UTF-8')
1022 output = output.split() 1038 output = output.split()
1023 if len(output) > 3: 1039 if len(output) > 3:
1024 return output[2] 1040 return output[2]
diff --git a/hooks/charmhelpers/contrib/storage/linux/lvm.py b/hooks/charmhelpers/contrib/storage/linux/lvm.py
index 4719f53..7f2a060 100644
--- a/hooks/charmhelpers/contrib/storage/linux/lvm.py
+++ b/hooks/charmhelpers/contrib/storage/linux/lvm.py
@@ -74,10 +74,10 @@ def list_lvm_volume_group(block_device):
74 ''' 74 '''
75 vg = None 75 vg = None
76 pvd = check_output(['pvdisplay', block_device]).splitlines() 76 pvd = check_output(['pvdisplay', block_device]).splitlines()
77 for l in pvd: 77 for lvm in pvd:
78 l = l.decode('UTF-8') 78 lvm = lvm.decode('UTF-8')
79 if l.strip().startswith('VG Name'): 79 if lvm.strip().startswith('VG Name'):
80 vg = ' '.join(l.strip().split()[2:]) 80 vg = ' '.join(lvm.strip().split()[2:])
81 return vg 81 return vg
82 82
83 83
diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py
index 3dc0df6..c942889 100644
--- a/hooks/charmhelpers/contrib/storage/linux/utils.py
+++ b/hooks/charmhelpers/contrib/storage/linux/utils.py
@@ -64,6 +64,6 @@ def is_device_mounted(device):
64 ''' 64 '''
65 try: 65 try:
66 out = check_output(['lsblk', '-P', device]).decode('UTF-8') 66 out = check_output(['lsblk', '-P', device]).decode('UTF-8')
67 except: 67 except Exception:
68 return False 68 return False
69 return bool(re.search(r'MOUNTPOINT=".+"', out)) 69 return bool(re.search(r'MOUNTPOINT=".+"', out))
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index 12f37b2..5a88f79 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -22,6 +22,7 @@ from __future__ import print_function
22import copy 22import copy
23from distutils.version import LooseVersion 23from distutils.version import LooseVersion
24from functools import wraps 24from functools import wraps
25from collections import namedtuple
25import glob 26import glob
26import os 27import os
27import json 28import json
@@ -218,6 +219,8 @@ def principal_unit():
218 for rid in relation_ids(reltype): 219 for rid in relation_ids(reltype):
219 for unit in related_units(rid): 220 for unit in related_units(rid):
220 md = _metadata_unit(unit) 221 md = _metadata_unit(unit)
222 if not md:
223 continue
221 subordinate = md.pop('subordinate', None) 224 subordinate = md.pop('subordinate', None)
222 if not subordinate: 225 if not subordinate:
223 return unit 226 return unit
@@ -511,7 +514,10 @@ def _metadata_unit(unit):
511 """ 514 """
512 basedir = os.sep.join(charm_dir().split(os.sep)[:-2]) 515 basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
513 unitdir = 'unit-{}'.format(unit.replace(os.sep, '-')) 516 unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
514 with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md: 517 joineddir = os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')
518 if not os.path.exists(joineddir):
519 return None
520 with open(joineddir) as md:
515 return yaml.safe_load(md) 521 return yaml.safe_load(md)
516 522
517 523
@@ -639,18 +645,31 @@ def is_relation_made(relation, keys='private-address'):
639 return False 645 return False
640 646
641 647
648def _port_op(op_name, port, protocol="TCP"):
649 """Open or close a service network port"""
650 _args = [op_name]
651 icmp = protocol.upper() == "ICMP"
652 if icmp:
653 _args.append(protocol)
654 else:
655 _args.append('{}/{}'.format(port, protocol))
656 try:
657 subprocess.check_call(_args)
658 except subprocess.CalledProcessError:
659 # Older Juju pre 2.3 doesn't support ICMP
660 # so treat it as a no-op if it fails.
661 if not icmp:
662 raise
663
664
642def open_port(port, protocol="TCP"): 665def open_port(port, protocol="TCP"):
643 """Open a service network port""" 666 """Open a service network port"""
644 _args = ['open-port'] 667 _port_op('open-port', port, protocol)
645 _args.append('{}/{}'.format(port, protocol))
646 subprocess.check_call(_args)
647 668
648 669
649def close_port(port, protocol="TCP"): 670def close_port(port, protocol="TCP"):
650 """Close a service network port""" 671 """Close a service network port"""
651 _args = ['close-port'] 672 _port_op('close-port', port, protocol)
652 _args.append('{}/{}'.format(port, protocol))
653 subprocess.check_call(_args)
654 673
655 674
656def open_ports(start, end, protocol="TCP"): 675def open_ports(start, end, protocol="TCP"):
@@ -667,6 +686,17 @@ def close_ports(start, end, protocol="TCP"):
667 subprocess.check_call(_args) 686 subprocess.check_call(_args)
668 687
669 688
689def opened_ports():
690 """Get the opened ports
691
692 *Note that this will only show ports opened in a previous hook*
693
694 :returns: Opened ports as a list of strings: ``['8080/tcp', '8081-8083/tcp']``
695 """
696 _args = ['opened-ports', '--format=json']
697 return json.loads(subprocess.check_output(_args).decode('UTF-8'))
698
699
670@cached 700@cached
671def unit_get(attribute): 701def unit_get(attribute):
672 """Get the unit ID for the remote unit""" 702 """Get the unit ID for the remote unit"""
@@ -1077,6 +1107,35 @@ def network_get_primary_address(binding):
1077 return subprocess.check_output(cmd).decode('UTF-8').strip() 1107 return subprocess.check_output(cmd).decode('UTF-8').strip()
1078 1108
1079 1109
1110@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
1111def network_get(endpoint, relation_id=None):
1112 """
1113 Retrieve the network details for a relation endpoint
1114
1115 :param endpoint: string. The name of a relation endpoint
1116 :param relation_id: int. The ID of the relation for the current context.
1117 :return: dict. The loaded YAML output of the network-get query.
1118 :raise: NotImplementedError if run on Juju < 2.1
1119 """
1120 cmd = ['network-get', endpoint, '--format', 'yaml']
1121 if relation_id:
1122 cmd.append('-r')
1123 cmd.append(relation_id)
1124 try:
1125 response = subprocess.check_output(
1126 cmd,
1127 stderr=subprocess.STDOUT).decode('UTF-8').strip()
1128 except CalledProcessError as e:
1129 # Early versions of Juju 2.0.x required the --primary-address argument.
1130 # We catch that condition here and raise NotImplementedError since
1131 # the requested semantics are not available - the caller can then
1132 # use the network_get_primary_address() method instead.
1133 if '--primary-address is currently required' in e.output.decode('UTF-8'):
1134 raise NotImplementedError
1135 raise
1136 return yaml.safe_load(response)
1137
1138
1080def add_metric(*args, **kwargs): 1139def add_metric(*args, **kwargs):
1081 """Add metric values. Values may be expressed with keyword arguments. For 1140 """Add metric values. Values may be expressed with keyword arguments. For
1082 metric names containing dashes, these may be expressed as one or more 1141 metric names containing dashes, these may be expressed as one or more
@@ -1106,3 +1165,42 @@ def meter_info():
1106 """Get the meter status information, if running in the meter-status-changed 1165 """Get the meter status information, if running in the meter-status-changed
1107 hook.""" 1166 hook."""
1108 return os.environ.get('JUJU_METER_INFO') 1167 return os.environ.get('JUJU_METER_INFO')
1168
1169
1170def iter_units_for_relation_name(relation_name):
1171 """Iterate through all units in a relation
1172
1173 Generator that iterates through all the units in a relation and yields
1174 a named tuple with rid and unit field names.
1175
1176 Usage:
1177 data = [(u.rid, u.unit)
1178 for u in iter_units_for_relation_name(relation_name)]
1179
1180 :param relation_name: string relation name
1181 :yield: Named Tuple with rid and unit field names
1182 """
1183 RelatedUnit = namedtuple('RelatedUnit', 'rid, unit')
1184 for rid in relation_ids(relation_name):
1185 for unit in related_units(rid):
1186 yield RelatedUnit(rid, unit)
1187
1188
1189def ingress_address(rid=None, unit=None):
1190 """
1191 Retrieve the ingress-address from a relation when available. Otherwise,
1192 return the private-address. This function is to be used on the consuming
1193 side of the relation.
1194
1195 Usage:
1196 addresses = [ingress_address(rid=u.rid, unit=u.unit)
1197 for u in iter_units_for_relation_name(relation_name)]
1198
1199 :param rid: string relation id
1200 :param unit: string unit name
1201 :side effect: calls relation_get
1202 :return: string IP address
1203 """
1204 settings = relation_get(rid=rid, unit=unit)
1205 return (settings.get('ingress-address') or
1206 settings.get('private-address'))
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index 5656e2f..5cc5c86 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -34,7 +34,7 @@ import six
34 34
35from contextlib import contextmanager 35from contextlib import contextmanager
36from collections import OrderedDict 36from collections import OrderedDict
37from .hookenv import log, DEBUG 37from .hookenv import log, DEBUG, local_unit
38from .fstab import Fstab 38from .fstab import Fstab
39from charmhelpers.osplatform import get_platform 39from charmhelpers.osplatform import get_platform
40 40
@@ -441,6 +441,49 @@ def add_user_to_group(username, group):
441 subprocess.check_call(cmd) 441 subprocess.check_call(cmd)
442 442
443 443
444def chage(username, lastday=None, expiredate=None, inactive=None,
445 mindays=None, maxdays=None, root=None, warndays=None):
446 """Change user password expiry information
447
448 :param str username: User to update
449 :param str lastday: Set when password was changed in YYYY-MM-DD format
450 :param str expiredate: Set when user's account will no longer be
451 accessible in YYYY-MM-DD format.
452 -1 will remove an account expiration date.
453 :param str inactive: Set the number of days of inactivity after a password
454 has expired before the account is locked.
455 -1 will remove an account's inactivity.
456 :param str mindays: Set the minimum number of days between password
457 changes to MIN_DAYS.
458 0 indicates the password can be changed anytime.
459 :param str maxdays: Set the maximum number of days during which a
460 password is valid.
461 -1 as MAX_DAYS will remove checking maxdays
462 :param str root: Apply changes in the CHROOT_DIR directory
463 :param str warndays: Set the number of days of warning before a password
464 change is required
465 :raises subprocess.CalledProcessError: if call to chage fails
466 """
467 cmd = ['chage']
468 if root:
469 cmd.extend(['--root', root])
470 if lastday:
471 cmd.extend(['--lastday', lastday])
472 if expiredate:
473 cmd.extend(['--expiredate', expiredate])
474 if inactive:
475 cmd.extend(['--inactive', inactive])
476 if mindays:
477 cmd.extend(['--mindays', mindays])
478 if maxdays:
479 cmd.extend(['--maxdays', maxdays])
480 if warndays:
481 cmd.extend(['--warndays', warndays])
482 cmd.append(username)
483 subprocess.check_call(cmd)
484
485remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
486
444def rsync(from_path, to_path, flags='-r', options=None, timeout=None): 487def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
445 """Replicate the contents of a path""" 488 """Replicate the contents of a path"""
446 options = options or ['--delete', '--executability'] 489 options = options or ['--delete', '--executability']
@@ -946,3 +989,31 @@ def updatedb(updatedb_text, new_path):
946 lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths)) 989 lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
947 output = "\n".join(lines) 990 output = "\n".join(lines)
948 return output 991 return output
992
993
994def modulo_distribution(modulo=3, wait=30):
995 """ Modulo distribution
996
997 This helper uses the unit number, a modulo value and a constant wait time
998 to produce a calculated wait time distribution. This is useful in large
999 scale deployments to distribute load during an expensive operation such as
1000 service restarts.
1001
1002 If you have 1000 nodes that need to restart 100 at a time 1 minute at a
1003 time:
1004
1005 time.wait(modulo_distribution(modulo=100, wait=60))
1006 restart()
1007
1008 If you need restarts to happen serially set modulo to the exact number of
1009 nodes and set a high constant wait time:
1010
1011 time.wait(modulo_distribution(modulo=10, wait=120))
1012 restart()
1013
1014 @param modulo: int The modulo number creates the group distribution
1015 @param wait: int The constant time wait value
1016 @return: int Calculated time to wait for unit operation
1017 """
1018 unit_number = int(local_unit().split('/')[1])
1019 return (unit_number % modulo) * wait
diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py
index 685dabd..e8df045 100644
--- a/hooks/charmhelpers/core/strutils.py
+++ b/hooks/charmhelpers/core/strutils.py
@@ -61,13 +61,19 @@ def bytes_from_string(value):
61 if isinstance(value, six.string_types): 61 if isinstance(value, six.string_types):
62 value = six.text_type(value) 62 value = six.text_type(value)
63 else: 63 else:
64 msg = "Unable to interpret non-string value '%s' as boolean" % (value) 64 msg = "Unable to interpret non-string value '%s' as bytes" % (value)
65 raise ValueError(msg) 65 raise ValueError(msg)
66 matches = re.match("([0-9]+)([a-zA-Z]+)", value) 66 matches = re.match("([0-9]+)([a-zA-Z]+)", value)
67 if not matches: 67 if matches:
68 msg = "Unable to interpret string value '%s' as bytes" % (value) 68 size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
69 raise ValueError(msg) 69 else:
70 return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) 70 # Assume that value passed in is bytes
71 try:
72 size = int(value)
73 except ValueError:
74 msg = "Unable to interpret string value '%s' as bytes" % (value)
75 raise ValueError(msg)
76 return size
71 77
72 78
73class BasicStringComparator(object): 79class BasicStringComparator(object):
diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py
index 54ec969..7af875c 100644
--- a/hooks/charmhelpers/core/unitdata.py
+++ b/hooks/charmhelpers/core/unitdata.py
@@ -358,7 +358,7 @@ class Storage(object):
358 try: 358 try:
359 yield self.revision 359 yield self.revision
360 self.revision = None 360 self.revision = None
361 except: 361 except Exception:
362 self.flush(False) 362 self.flush(False)
363 self.revision = None 363 self.revision = None
364 raise 364 raise
diff --git a/hooks/charmhelpers/fetch/snap.py b/hooks/charmhelpers/fetch/snap.py
index 112a54c..395836c 100644
--- a/hooks/charmhelpers/fetch/snap.py
+++ b/hooks/charmhelpers/fetch/snap.py
@@ -41,6 +41,10 @@ class CouldNotAcquireLockException(Exception):
41 pass 41 pass
42 42
43 43
44class InvalidSnapChannel(Exception):
45 pass
46
47
44def _snap_exec(commands): 48def _snap_exec(commands):
45 """ 49 """
46 Execute snap commands. 50 Execute snap commands.
@@ -132,3 +136,15 @@ def snap_refresh(packages, *flags):
132 136
133 log(message, level='INFO') 137 log(message, level='INFO')
134 return _snap_exec(['refresh'] + flags + packages) 138 return _snap_exec(['refresh'] + flags + packages)
139
140
141def valid_snap_channel(channel):
142 """ Validate snap channel exists
143
144 :raises InvalidSnapChannel: When channel does not exist
145 :return: Boolean
146 """
147 if channel.lower() in SNAP_CHANNELS:
148 return True
149 else:
150 raise InvalidSnapChannel("Invalid Snap Channel: {}".format(channel))
diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py
index 40e1cb5..910e96a 100644
--- a/hooks/charmhelpers/fetch/ubuntu.py
+++ b/hooks/charmhelpers/fetch/ubuntu.py
@@ -572,7 +572,7 @@ def get_upstream_version(package):
572 cache = apt_cache() 572 cache = apt_cache()
573 try: 573 try:
574 pkg = cache[package] 574 pkg = cache[package]
575 except: 575 except Exception:
576 # the package is unknown to the current apt cache. 576 # the package is unknown to the current apt cache.
577 return None 577 return None
578 578
diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index 70c7e43..04ff28e 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -782,17 +782,17 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment):
782 782
783 name = "demo-vol" 783 name = "demo-vol"
784 vols = self.cinder.volumes.list() 784 vols = self.cinder.volumes.list()
785 cinder_vols = [v for v in vols if v.display_name == name] 785 cinder_vols = [v for v in vols if v.name == name]
786 if not cinder_vols: 786 if not cinder_vols:
787 # NOTE(hopem): it appears that at some point cinder-backup stopped 787 # NOTE(hopem): it appears that at some point cinder-backup stopped
788 # restoring volume metadata properly so revert to default name if 788 # restoring volume metadata properly so revert to default name if
789 # original is not found 789 # original is not found
790 name = "restore_backup_%s" % (vol_backup.id) 790 name = "restore_backup_%s" % (vol_backup.id)
791 cinder_vols = [v for v in vols if v.display_name == name] 791 cinder_vols = [v for v in vols if v.name == name]
792 792
793 if not cinder_vols: 793 if not cinder_vols:
794 msg = ("Could not find restore vol '%s' in %s" % 794 msg = ("Could not find restore vol '%s' in %s" %
795 (name, [v.display_name for v in vols])) 795 (name, [v.name for v in vols]))
796 u.log.error(msg) 796 u.log.error(msg)
797 amulet.raise_status(amulet.FAIL, msg=msg) 797 amulet.raise_status(amulet.FAIL, msg=msg)
798 798
diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
index 5c041d2..5afbbd8 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -13,6 +13,7 @@
13# limitations under the License. 13# limitations under the License.
14 14
15import logging 15import logging
16import os
16import re 17import re
17import sys 18import sys
18import six 19import six
@@ -185,7 +186,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
185 self.d.configure(service, config) 186 self.d.configure(service, config)
186 187
187 def _auto_wait_for_status(self, message=None, exclude_services=None, 188 def _auto_wait_for_status(self, message=None, exclude_services=None,
188 include_only=None, timeout=1800): 189 include_only=None, timeout=None):
189 """Wait for all units to have a specific extended status, except 190 """Wait for all units to have a specific extended status, except
190 for any defined as excluded. Unless specified via message, any 191 for any defined as excluded. Unless specified via message, any
191 status containing any case of 'ready' will be considered a match. 192 status containing any case of 'ready' will be considered a match.
@@ -215,7 +216,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
215 :param timeout: Maximum time in seconds to wait for status match 216 :param timeout: Maximum time in seconds to wait for status match
216 :returns: None. Raises if timeout is hit. 217 :returns: None. Raises if timeout is hit.
217 """ 218 """
218 self.log.info('Waiting for extended status on units...') 219 if not timeout:
220 timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 1800))
221 self.log.info('Waiting for extended status on units for {}s...'
222 ''.format(timeout))
219 223
220 all_services = self.d.services.keys() 224 all_services = self.d.services.keys()
221 225
@@ -250,7 +254,14 @@ class OpenStackAmuletDeployment(AmuletDeployment):
250 self.log.debug('Waiting up to {}s for extended status on services: ' 254 self.log.debug('Waiting up to {}s for extended status on services: '
251 '{}'.format(timeout, services)) 255 '{}'.format(timeout, services))
252 service_messages = {service: message for service in services} 256 service_messages = {service: message for service in services}
257
258 # Check for idleness
259 self.d.sentry.wait(timeout=timeout)
260 # Check for error states and bail early
261 self.d.sentry.wait_for_status(self.d.juju_env, services, timeout=timeout)
262 # Check for ready messages
253 self.d.sentry.wait_for_messages(service_messages, timeout=timeout) 263 self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
264
254 self.log.info('OK') 265 self.log.info('OK')
255 266
256 def _get_openstack_release(self): 267 def _get_openstack_release(self):
@@ -263,7 +274,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
263 (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty, 274 (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
264 self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton, 275 self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
265 self.yakkety_newton, self.xenial_ocata, self.zesty_ocata, 276 self.yakkety_newton, self.xenial_ocata, self.zesty_ocata,
266 self.xenial_pike, self.artful_pike) = range(11) 277 self.xenial_pike, self.artful_pike, self.xenial_queens,
278 self.bionic_queens,) = range(13)
267 279
268 releases = { 280 releases = {
269 ('trusty', None): self.trusty_icehouse, 281 ('trusty', None): self.trusty_icehouse,
@@ -274,9 +286,11 @@ class OpenStackAmuletDeployment(AmuletDeployment):
274 ('xenial', 'cloud:xenial-newton'): self.xenial_newton, 286 ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
275 ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata, 287 ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
276 ('xenial', 'cloud:xenial-pike'): self.xenial_pike, 288 ('xenial', 'cloud:xenial-pike'): self.xenial_pike,
289 ('xenial', 'cloud:xenial-queens'): self.xenial_queens,
277 ('yakkety', None): self.yakkety_newton, 290 ('yakkety', None): self.yakkety_newton,
278 ('zesty', None): self.zesty_ocata, 291 ('zesty', None): self.zesty_ocata,
279 ('artful', None): self.artful_pike, 292 ('artful', None): self.artful_pike,
293 ('bionic', None): self.bionic_queens,
280 } 294 }
281 return releases[(self.series, self.openstack)] 295 return releases[(self.series, self.openstack)]
282 296
@@ -291,6 +305,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
291 ('yakkety', 'newton'), 305 ('yakkety', 'newton'),
292 ('zesty', 'ocata'), 306 ('zesty', 'ocata'),
293 ('artful', 'pike'), 307 ('artful', 'pike'),
308 ('bionic', 'queens'),
294 ]) 309 ])
295 if self.openstack: 310 if self.openstack:
296 os_origin = self.openstack.split(':')[1] 311 os_origin = self.openstack.split(':')[1]
@@ -303,20 +318,27 @@ class OpenStackAmuletDeployment(AmuletDeployment):
303 test scenario, based on OpenStack release and whether ceph radosgw 318 test scenario, based on OpenStack release and whether ceph radosgw
304 is flagged as present or not.""" 319 is flagged as present or not."""
305 320
306 if self._get_openstack_release() >= self.trusty_kilo: 321 if self._get_openstack_release() == self.trusty_icehouse:
307 # Kilo or later 322 # Icehouse
308 pools = [ 323 pools = [
324 'data',
325 'metadata',
309 'rbd', 326 'rbd',
310 'cinder', 327 'cinder-ceph',
311 'glance' 328 'glance'
312 ] 329 ]
313 else: 330 elif (self.trusty_kilo <= self._get_openstack_release() <=
314 # Juno or earlier 331 self.zesty_ocata):
332 # Kilo through Ocata
315 pools = [ 333 pools = [
316 'data',
317 'metadata',
318 'rbd', 334 'rbd',
319 'cinder', 335 'cinder-ceph',
336 'glance'
337 ]
338 else:
339 # Pike and later
340 pools = [
341 'cinder-ceph',
320 'glance' 342 'glance'
321 ] 343 ]
322 344
diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py
index c8edbf6..b71b2b1 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py
@@ -23,6 +23,7 @@ import urllib
23import urlparse 23import urlparse
24 24
25import cinderclient.v1.client as cinder_client 25import cinderclient.v1.client as cinder_client
26import cinderclient.v2.client as cinder_clientv2
26import glanceclient.v1.client as glance_client 27import glanceclient.v1.client as glance_client
27import heatclient.v1.client as heat_client 28import heatclient.v1.client as heat_client
28from keystoneclient.v2_0 import client as keystone_client 29from keystoneclient.v2_0 import client as keystone_client
@@ -42,7 +43,6 @@ import swiftclient
42from charmhelpers.contrib.amulet.utils import ( 43from charmhelpers.contrib.amulet.utils import (
43 AmuletUtils 44 AmuletUtils
44) 45)
45from charmhelpers.core.decorators import retry_on_exception
46from charmhelpers.core.host import CompareHostReleases 46from charmhelpers.core.host import CompareHostReleases
47 47
48DEBUG = logging.DEBUG 48DEBUG = logging.DEBUG
@@ -310,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils):
310 self.log.debug('Checking if tenant exists ({})...'.format(tenant)) 310 self.log.debug('Checking if tenant exists ({})...'.format(tenant))
311 return tenant in [t.name for t in keystone.tenants.list()] 311 return tenant in [t.name for t in keystone.tenants.list()]
312 312
313 @retry_on_exception(5, base_delay=10)
314 def keystone_wait_for_propagation(self, sentry_relation_pairs, 313 def keystone_wait_for_propagation(self, sentry_relation_pairs,
315 api_version): 314 api_version):
316 """Iterate over list of sentry and relation tuples and verify that 315 """Iterate over list of sentry and relation tuples and verify that
@@ -326,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils):
326 rel = sentry.relation('identity-service', 325 rel = sentry.relation('identity-service',
327 relation_name) 326 relation_name)
328 self.log.debug('keystone relation data: {}'.format(rel)) 327 self.log.debug('keystone relation data: {}'.format(rel))
329 if rel['api_version'] != str(api_version): 328 if rel.get('api_version') != str(api_version):
330 raise Exception("api_version not propagated through relation" 329 raise Exception("api_version not propagated through relation"
331 " data yet ('{}' != '{}')." 330 " data yet ('{}' != '{}')."
332 "".format(rel['api_version'], api_version)) 331 "".format(rel['api_version'], api_version))
@@ -348,15 +347,19 @@ class OpenStackAmuletUtils(AmuletUtils):
348 347
349 config = {'preferred-api-version': api_version} 348 config = {'preferred-api-version': api_version}
350 deployment.d.configure('keystone', config) 349 deployment.d.configure('keystone', config)
350 deployment._auto_wait_for_status()
351 self.keystone_wait_for_propagation(sentry_relation_pairs, api_version) 351 self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
352 352
353 def authenticate_cinder_admin(self, keystone_sentry, username, 353 def authenticate_cinder_admin(self, keystone_sentry, username,
354 password, tenant): 354 password, tenant, api_version=2):
355 """Authenticates admin user with cinder.""" 355 """Authenticates admin user with cinder."""
356 # NOTE(beisner): cinder python client doesn't accept tokens. 356 # NOTE(beisner): cinder python client doesn't accept tokens.
357 keystone_ip = keystone_sentry.info['public-address'] 357 keystone_ip = keystone_sentry.info['public-address']
358 ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8')) 358 ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
359 return cinder_client.Client(username, password, tenant, ept) 359 _clients = {
360 1: cinder_client.Client,
361 2: cinder_clientv2.Client}
362 return _clients[api_version](username, password, tenant, ept)
360 363
361 def authenticate_keystone(self, keystone_ip, username, password, 364 def authenticate_keystone(self, keystone_ip, username, password,
362 api_version=False, admin_port=False, 365 api_version=False, admin_port=False,
@@ -617,13 +620,25 @@ class OpenStackAmuletUtils(AmuletUtils):
617 self.log.debug('Keypair ({}) already exists, ' 620 self.log.debug('Keypair ({}) already exists, '
618 'using it.'.format(keypair_name)) 621 'using it.'.format(keypair_name))
619 return _keypair 622 return _keypair
620 except: 623 except Exception:
621 self.log.debug('Keypair ({}) does not exist, ' 624 self.log.debug('Keypair ({}) does not exist, '
622 'creating it.'.format(keypair_name)) 625 'creating it.'.format(keypair_name))
623 626
624 _keypair = nova.keypairs.create(name=keypair_name) 627 _keypair = nova.keypairs.create(name=keypair_name)
625 return _keypair 628 return _keypair
626 629
630 def _get_cinder_obj_name(self, cinder_object):
631 """Retrieve name of cinder object.
632
633 :param cinder_object: cinder snapshot or volume object
634 :returns: str cinder object name
635 """
636 # v1 objects store name in 'display_name' attr but v2+ use 'name'
637 try:
638 return cinder_object.display_name
639 except AttributeError:
640 return cinder_object.name
641
627 def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1, 642 def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
628 img_id=None, src_vol_id=None, snap_id=None): 643 img_id=None, src_vol_id=None, snap_id=None):
629 """Create cinder volume, optionally from a glance image, OR 644 """Create cinder volume, optionally from a glance image, OR
@@ -674,6 +689,13 @@ class OpenStackAmuletUtils(AmuletUtils):
674 source_volid=src_vol_id, 689 source_volid=src_vol_id,
675 snapshot_id=snap_id) 690 snapshot_id=snap_id)
676 vol_id = vol_new.id 691 vol_id = vol_new.id
692 except TypeError:
693 vol_new = cinder.volumes.create(name=vol_name,
694 imageRef=img_id,
695 size=vol_size,
696 source_volid=src_vol_id,
697 snapshot_id=snap_id)
698 vol_id = vol_new.id
677 except Exception as e: 699 except Exception as e:
678 msg = 'Failed to create volume: {}'.format(e) 700 msg = 'Failed to create volume: {}'.format(e)
679 amulet.raise_status(amulet.FAIL, msg=msg) 701 amulet.raise_status(amulet.FAIL, msg=msg)
@@ -688,7 +710,7 @@ class OpenStackAmuletUtils(AmuletUtils):
688 710
689 # Re-validate new volume 711 # Re-validate new volume
690 self.log.debug('Validating volume attributes...') 712 self.log.debug('Validating volume attributes...')
691 val_vol_name = cinder.volumes.get(vol_id).display_name 713 val_vol_name = self._get_cinder_obj_name(cinder.volumes.get(vol_id))
692 val_vol_boot = cinder.volumes.get(vol_id).bootable 714 val_vol_boot = cinder.volumes.get(vol_id).bootable
693 val_vol_stat = cinder.volumes.get(vol_id).status 715 val_vol_stat = cinder.volumes.get(vol_id).status
694 val_vol_size = cinder.volumes.get(vol_id).size 716 val_vol_size = cinder.volumes.get(vol_id).size
diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py
index 12f37b2..5a88f79 100644
--- a/tests/charmhelpers/core/hookenv.py
+++ b/tests/charmhelpers/core/hookenv.py
@@ -22,6 +22,7 @@ from __future__ import print_function
22import copy 22import copy
23from distutils.version import LooseVersion 23from distutils.version import LooseVersion
24from functools import wraps 24from functools import wraps
25from collections import namedtuple
25import glob 26import glob
26import os 27import os
27import json 28import json
@@ -218,6 +219,8 @@ def principal_unit():
218 for rid in relation_ids(reltype): 219 for rid in relation_ids(reltype):
219 for unit in related_units(rid): 220 for unit in related_units(rid):
220 md = _metadata_unit(unit) 221 md = _metadata_unit(unit)
222 if not md:
223 continue
221 subordinate = md.pop('subordinate', None) 224 subordinate = md.pop('subordinate', None)
222 if not subordinate: 225 if not subordinate:
223 return unit 226 return unit
@@ -511,7 +514,10 @@ def _metadata_unit(unit):
511 """ 514 """
512 basedir = os.sep.join(charm_dir().split(os.sep)[:-2]) 515 basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
513 unitdir = 'unit-{}'.format(unit.replace(os.sep, '-')) 516 unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
514 with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md: 517 joineddir = os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')
518 if not os.path.exists(joineddir):
519 return None
520 with open(joineddir) as md:
515 return yaml.safe_load(md) 521 return yaml.safe_load(md)
516 522
517 523
@@ -639,18 +645,31 @@ def is_relation_made(relation, keys='private-address'):
639 return False 645 return False
640 646
641 647
648def _port_op(op_name, port, protocol="TCP"):
649 """Open or close a service network port"""
650 _args = [op_name]
651 icmp = protocol.upper() == "ICMP"
652 if icmp:
653 _args.append(protocol)
654 else:
655 _args.append('{}/{}'.format(port, protocol))
656 try:
657 subprocess.check_call(_args)
658 except subprocess.CalledProcessError:
659 # Older Juju pre 2.3 doesn't support ICMP
660 # so treat it as a no-op if it fails.
661 if not icmp:
662 raise
663
664
642def open_port(port, protocol="TCP"): 665def open_port(port, protocol="TCP"):
643 """Open a service network port""" 666 """Open a service network port"""
644 _args = ['open-port'] 667 _port_op('open-port', port, protocol)
645 _args.append('{}/{}'.format(port, protocol))
646 subprocess.check_call(_args)
647 668
648 669
649def close_port(port, protocol="TCP"): 670def close_port(port, protocol="TCP"):
650 """Close a service network port""" 671 """Close a service network port"""
651 _args = ['close-port'] 672 _port_op('close-port', port, protocol)
652 _args.append('{}/{}'.format(port, protocol))
653 subprocess.check_call(_args)
654 673
655 674
656def open_ports(start, end, protocol="TCP"): 675def open_ports(start, end, protocol="TCP"):
@@ -667,6 +686,17 @@ def close_ports(start, end, protocol="TCP"):
667 subprocess.check_call(_args) 686 subprocess.check_call(_args)
668 687
669 688
689def opened_ports():
690 """Get the opened ports
691
692 *Note that this will only show ports opened in a previous hook*
693
694 :returns: Opened ports as a list of strings: ``['8080/tcp', '8081-8083/tcp']``
695 """
696 _args = ['opened-ports', '--format=json']
697 return json.loads(subprocess.check_output(_args).decode('UTF-8'))
698
699
670@cached 700@cached
671def unit_get(attribute): 701def unit_get(attribute):
672 """Get the unit ID for the remote unit""" 702 """Get the unit ID for the remote unit"""
@@ -1077,6 +1107,35 @@ def network_get_primary_address(binding):
1077 return subprocess.check_output(cmd).decode('UTF-8').strip() 1107 return subprocess.check_output(cmd).decode('UTF-8').strip()
1078 1108
1079 1109
1110@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
1111def network_get(endpoint, relation_id=None):
1112 """
1113 Retrieve the network details for a relation endpoint
1114
1115 :param endpoint: string. The name of a relation endpoint
1116 :param relation_id: int. The ID of the relation for the current context.
1117 :return: dict. The loaded YAML output of the network-get query.
1118 :raise: NotImplementedError if run on Juju < 2.1
1119 """
1120 cmd = ['network-get', endpoint, '--format', 'yaml']
1121 if relation_id:
1122 cmd.append('-r')
1123 cmd.append(relation_id)
1124 try:
1125 response = subprocess.check_output(
1126 cmd,
1127 stderr=subprocess.STDOUT).decode('UTF-8').strip()
1128 except CalledProcessError as e:
1129 # Early versions of Juju 2.0.x required the --primary-address argument.
1130 # We catch that condition here and raise NotImplementedError since
1131 # the requested semantics are not available - the caller can then
1132 # use the network_get_primary_address() method instead.
1133 if '--primary-address is currently required' in e.output.decode('UTF-8'):
1134 raise NotImplementedError
1135 raise
1136 return yaml.safe_load(response)
1137
1138
1080def add_metric(*args, **kwargs): 1139def add_metric(*args, **kwargs):
1081 """Add metric values. Values may be expressed with keyword arguments. For 1140 """Add metric values. Values may be expressed with keyword arguments. For
1082 metric names containing dashes, these may be expressed as one or more 1141 metric names containing dashes, these may be expressed as one or more
@@ -1106,3 +1165,42 @@ def meter_info():
1106 """Get the meter status information, if running in the meter-status-changed 1165 """Get the meter status information, if running in the meter-status-changed
1107 hook.""" 1166 hook."""
1108 return os.environ.get('JUJU_METER_INFO') 1167 return os.environ.get('JUJU_METER_INFO')
1168
1169
1170def iter_units_for_relation_name(relation_name):
1171 """Iterate through all units in a relation
1172
1173 Generator that iterates through all the units in a relation and yields
1174 a named tuple with rid and unit field names.
1175
1176 Usage:
1177 data = [(u.rid, u.unit)
1178 for u in iter_units_for_relation_name(relation_name)]
1179
1180 :param relation_name: string relation name
1181 :yield: Named Tuple with rid and unit field names
1182 """
1183 RelatedUnit = namedtuple('RelatedUnit', 'rid, unit')
1184 for rid in relation_ids(relation_name):
1185 for unit in related_units(rid):
1186 yield RelatedUnit(rid, unit)
1187
1188
1189def ingress_address(rid=None, unit=None):
1190 """
1191 Retrieve the ingress-address from a relation when available. Otherwise,
1192 return the private-address. This function is to be used on the consuming
1193 side of the relation.
1194
1195 Usage:
1196 addresses = [ingress_address(rid=u.rid, unit=u.unit)
1197 for u in iter_units_for_relation_name(relation_name)]
1198
1199 :param rid: string relation id
1200 :param unit: string unit name
1201 :side effect: calls relation_get
1202 :return: string IP address
1203 """
1204 settings = relation_get(rid=rid, unit=unit)
1205 return (settings.get('ingress-address') or
1206 settings.get('private-address'))
diff --git a/tests/charmhelpers/core/host.py b/tests/charmhelpers/core/host.py
index 5656e2f..5cc5c86 100644
--- a/tests/charmhelpers/core/host.py
+++ b/tests/charmhelpers/core/host.py
@@ -34,7 +34,7 @@ import six
34 34
35from contextlib import contextmanager 35from contextlib import contextmanager
36from collections import OrderedDict 36from collections import OrderedDict
37from .hookenv import log, DEBUG 37from .hookenv import log, DEBUG, local_unit
38from .fstab import Fstab 38from .fstab import Fstab
39from charmhelpers.osplatform import get_platform 39from charmhelpers.osplatform import get_platform
40 40
@@ -441,6 +441,49 @@ def add_user_to_group(username, group):
441 subprocess.check_call(cmd) 441 subprocess.check_call(cmd)
442 442
443 443
444def chage(username, lastday=None, expiredate=None, inactive=None,
445 mindays=None, maxdays=None, root=None, warndays=None):
446 """Change user password expiry information
447
448 :param str username: User to update
449 :param str lastday: Set when password was changed in YYYY-MM-DD format
450 :param str expiredate: Set when user's account will no longer be
451 accessible in YYYY-MM-DD format.
452 -1 will remove an account expiration date.
453 :param str inactive: Set the number of days of inactivity after a password
454 has expired before the account is locked.
455 -1 will remove an account's inactivity.
456 :param str mindays: Set the minimum number of days between password
457 changes to MIN_DAYS.
458 0 indicates the password can be changed anytime.
459 :param str maxdays: Set the maximum number of days during which a
460 password is valid.
461 -1 as MAX_DAYS will remove checking maxdays
462 :param str root: Apply changes in the CHROOT_DIR directory
463 :param str warndays: Set the number of days of warning before a password
464 change is required
465 :raises subprocess.CalledProcessError: if call to chage fails
466 """
467 cmd = ['chage']
468 if root:
469 cmd.extend(['--root', root])
470 if lastday:
471 cmd.extend(['--lastday', lastday])
472 if expiredate:
473 cmd.extend(['--expiredate', expiredate])
474 if inactive:
475 cmd.extend(['--inactive', inactive])
476 if mindays:
477 cmd.extend(['--mindays', mindays])
478 if maxdays:
479 cmd.extend(['--maxdays', maxdays])
480 if warndays:
481 cmd.extend(['--warndays', warndays])
482 cmd.append(username)
483 subprocess.check_call(cmd)
484
485remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
486
444def rsync(from_path, to_path, flags='-r', options=None, timeout=None): 487def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
445 """Replicate the contents of a path""" 488 """Replicate the contents of a path"""
446 options = options or ['--delete', '--executability'] 489 options = options or ['--delete', '--executability']
@@ -946,3 +989,31 @@ def updatedb(updatedb_text, new_path):
946 lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths)) 989 lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
947 output = "\n".join(lines) 990 output = "\n".join(lines)
948 return output 991 return output
992
993
994def modulo_distribution(modulo=3, wait=30):
995 """ Modulo distribution
996
997 This helper uses the unit number, a modulo value and a constant wait time
998 to produce a calculated wait time distribution. This is useful in large
999 scale deployments to distribute load during an expensive operation such as
1000 service restarts.
1001
1002 If you have 1000 nodes that need to restart 100 at a time 1 minute at a
1003 time:
1004
1005 time.wait(modulo_distribution(modulo=100, wait=60))
1006 restart()
1007
1008 If you need restarts to happen serially set modulo to the exact number of
1009 nodes and set a high constant wait time:
1010
1011 time.wait(modulo_distribution(modulo=10, wait=120))
1012 restart()
1013
1014 @param modulo: int The modulo number creates the group distribution
1015 @param wait: int The constant time wait value
1016 @return: int Calculated time to wait for unit operation
1017 """
1018 unit_number = int(local_unit().split('/')[1])
1019 return (unit_number % modulo) * wait
diff --git a/tests/charmhelpers/core/strutils.py b/tests/charmhelpers/core/strutils.py
index 685dabd..e8df045 100644
--- a/tests/charmhelpers/core/strutils.py
+++ b/tests/charmhelpers/core/strutils.py
@@ -61,13 +61,19 @@ def bytes_from_string(value):
61 if isinstance(value, six.string_types): 61 if isinstance(value, six.string_types):
62 value = six.text_type(value) 62 value = six.text_type(value)
63 else: 63 else:
64 msg = "Unable to interpret non-string value '%s' as boolean" % (value) 64 msg = "Unable to interpret non-string value '%s' as bytes" % (value)
65 raise ValueError(msg) 65 raise ValueError(msg)
66 matches = re.match("([0-9]+)([a-zA-Z]+)", value) 66 matches = re.match("([0-9]+)([a-zA-Z]+)", value)
67 if not matches: 67 if matches:
68 msg = "Unable to interpret string value '%s' as bytes" % (value) 68 size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
69 raise ValueError(msg) 69 else:
70 return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) 70 # Assume that value passed in is bytes
71 try:
72 size = int(value)
73 except ValueError:
74 msg = "Unable to interpret string value '%s' as bytes" % (value)
75 raise ValueError(msg)
76 return size
71 77
72 78
73class BasicStringComparator(object): 79class BasicStringComparator(object):
diff --git a/tests/charmhelpers/core/unitdata.py b/tests/charmhelpers/core/unitdata.py
index 54ec969..7af875c 100644
--- a/tests/charmhelpers/core/unitdata.py
+++ b/tests/charmhelpers/core/unitdata.py
@@ -358,7 +358,7 @@ class Storage(object):
358 try: 358 try:
359 yield self.revision 359 yield self.revision
360 self.revision = None 360 self.revision = None
361 except: 361 except Exception:
362 self.flush(False) 362 self.flush(False)
363 self.revision = None 363 self.revision = None
364 raise 364 raise