Decode subprocess output for python3 in ipv6 check

When using python3, the code encounters errors related to type
mismatch. subprocess requires an extra decode of return values
to address this.

CH Sync to get fix.

Change-Id: I141c0d4d1851012d5c0a5a606402c75211a323f5
Partial-Bug: #1691881
This commit is contained in:
David Ames 2017-05-31 15:27:04 -07:00
parent 851b1812f5
commit 36a4edff9c
10 changed files with 461 additions and 202 deletions

View File

@ -14,6 +14,11 @@
# Bootstrap charm-helpers, installing its dependencies if necessary using
# only standard libraries.
from __future__ import print_function
from __future__ import absolute_import
import functools
import inspect
import subprocess
import sys
@ -34,3 +39,59 @@ except ImportError:
else:
subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
import yaml # flake8: noqa
# Holds a list of mapping of mangled function names that have been deprecated
# using the @deprecate decorator below. This is so that the warning is only
# printed once for each usage of the function.
__deprecated_functions = {}
def deprecate(warning, date=None, log=None):
"""Add a deprecation warning the first time the function is used.
The date, which is a string in semi-ISO8660 format indicate the year-month
that the function is officially going to be removed.
usage:
@deprecate('use core/fetch/add_source() instead', '2017-04')
def contributed_add_source_thing(...):
...
And it then prints to the log ONCE that the function is deprecated.
The reason for passing the logging function (log) is so that hookenv.log
can be used for a charm if needed.
:param warning: String to indicat where it has moved ot.
:param date: optional sting, in YYYY-MM format to indicate when the
function will definitely (probably) be removed.
:param log: The log function to call to log. If not, logs to stdout
"""
def wrap(f):
@functools.wraps(f)
def wrapped_f(*args, **kwargs):
try:
module = inspect.getmodule(f)
file = inspect.getsourcefile(f)
lines = inspect.getsourcelines(f)
f_name = "{}-{}-{}..{}-{}".format(
module.__name__, file, lines[0], lines[-1], f.__name__)
except (IOError, TypeError):
# assume it was local, so just use the name of the function
f_name = f.__name__
if f_name not in __deprecated_functions:
__deprecated_functions[f_name] = True
s = "DEPRECATION WARNING: Function {} is being removed".format(
f.__name__)
if date:
s = "{} on/around {}".format(s, date)
if warning:
s = "{} : {}".format(s, warning)
if log:
log(s)
else:
print(s)
return f(*args, **kwargs)
return wrapped_f
return wrap

View File

@ -193,6 +193,13 @@ define service {{
nrpe_check_file = self._get_check_filename()
with open(nrpe_check_file, 'w') as nrpe_check_config:
nrpe_check_config.write("# check {}\n".format(self.shortname))
if nagios_servicegroups:
nrpe_check_config.write(
"# The following header was added automatically by juju\n")
nrpe_check_config.write(
"# Modifying it will affect nagios monitoring and alerting\n")
nrpe_check_config.write(
"# servicegroups: {}\n".format(nagios_servicegroups))
nrpe_check_config.write("command[{}]={}\n".format(
self.command, self.check_cmd))

View File

@ -243,7 +243,7 @@ def is_ipv6_disabled():
try:
result = subprocess.check_output(
['sysctl', 'net.ipv6.conf.all.disable_ipv6'],
stderr=subprocess.STDOUT)
stderr=subprocess.STDOUT).decode('UTF-8')
return "net.ipv6.conf.all.disable_ipv6 = 1" in result
except subprocess.CalledProcessError:
return True

View File

@ -26,11 +26,12 @@ import functools
import shutil
import six
import tempfile
import traceback
import uuid
import yaml
from charmhelpers import deprecate
from charmhelpers.contrib.network import ip
from charmhelpers.core import unitdata
@ -41,7 +42,6 @@ from charmhelpers.core.hookenv import (
config,
log as juju_log,
charm_dir,
DEBUG,
INFO,
ERROR,
related_units,
@ -82,9 +82,12 @@ from charmhelpers.core.host import (
restart_on_change_helper,
)
from charmhelpers.fetch import (
apt_install,
apt_cache,
install_remote,
import_key as fetch_import_key,
add_source as fetch_add_source,
SourceConfigError,
GPGKeyError,
get_upstream_version
)
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
@ -469,13 +472,14 @@ def get_os_version_package(pkg, fatal=True):
# error_out(e)
os_rel = None
# Module local cache variable for the os_release.
_os_rel = None
def reset_os_release():
'''Unset the cached os_release version'''
global os_rel
os_rel = None
global _os_rel
_os_rel = None
def os_release(package, base='essex', reset_cache=False):
@ -489,150 +493,77 @@ def os_release(package, base='essex', reset_cache=False):
the installation source, the earliest release supported by the charm should
be returned.
'''
global os_rel
global _os_rel
if reset_cache:
reset_os_release()
if os_rel:
return os_rel
os_rel = (git_os_codename_install_source(config('openstack-origin-git')) or
get_os_codename_package(package, fatal=False) or
get_os_codename_install_source(config('openstack-origin')) or
base)
return os_rel
if _os_rel:
return _os_rel
_os_rel = (
git_os_codename_install_source(config('openstack-origin-git')) or
get_os_codename_package(package, fatal=False) or
get_os_codename_install_source(config('openstack-origin')) or
base)
return _os_rel
@deprecate("moved to charmhelpers.fetch.import_key()", "2017-07", log=juju_log)
def import_key(keyid):
key = keyid.strip()
if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and
key.endswith('-----END PGP PUBLIC KEY BLOCK-----')):
juju_log("PGP key found (looks like ASCII Armor format)", level=DEBUG)
juju_log("Importing ASCII Armor PGP key", level=DEBUG)
with tempfile.NamedTemporaryFile() as keyfile:
with open(keyfile.name, 'w') as fd:
fd.write(key)
fd.write("\n")
"""Import a key, either ASCII armored, or a GPG key id.
cmd = ['apt-key', 'add', keyfile.name]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
error_out("Error importing PGP key '%s'" % key)
else:
juju_log("PGP key found (looks like Radix64 format)", level=DEBUG)
juju_log("Importing PGP key from keyserver", level=DEBUG)
cmd = ['apt-key', 'adv', '--keyserver',
'hkp://keyserver.ubuntu.com:80', '--recv-keys', key]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
error_out("Error importing PGP key '%s'" % key)
@param keyid: the key in ASCII armor format, or a GPG key id.
@raises SystemExit() via sys.exit() on failure.
"""
try:
return fetch_import_key(keyid)
except GPGKeyError as e:
error_out("Could not import key: {}".format(str(e)))
def get_source_and_pgp_key(input):
"""Look for a pgp key ID or ascii-armor key in the given input."""
index = input.strip()
index = input.rfind('|')
if index < 0:
return input, None
def get_source_and_pgp_key(source_and_key):
"""Look for a pgp key ID or ascii-armor key in the given input.
key = input[index + 1:].strip('|')
source = input[:index]
return source, key
:param source_and_key: Sting, "source_spec|keyid" where '|keyid' is
optional.
:returns (source_spec, key_id OR None) as a tuple. Returns None for key_id
if there was no '|' in the source_and_key string.
"""
try:
source, key = source_and_key.split('|', 2)
return source, key or None
except ValueError:
return source_and_key, None
def configure_installation_source(rel):
'''Configure apt installation source.'''
if rel == 'distro':
return
elif rel == 'distro-proposed':
ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f:
f.write(DISTRO_PROPOSED % ubuntu_rel)
elif rel[:4] == "ppa:":
src, key = get_source_and_pgp_key(rel)
if key:
import_key(key)
@deprecate("use charmhelpers.fetch.add_source() instead.",
"2017-07", log=juju_log)
def configure_installation_source(source_plus_key):
"""Configure an installation source.
subprocess.check_call(["add-apt-repository", "-y", src])
elif rel[:3] == "deb":
src, key = get_source_and_pgp_key(rel)
if key:
import_key(key)
The functionality is provided by charmhelpers.fetch.add_source()
The difference between the two functions is that add_source() signature
requires the key to be passed directly, whereas this function passes an
optional key by appending '|<key>' to the end of the source specificiation
'source'.
with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f:
f.write(src)
elif rel[:6] == 'cloud:':
ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
rel = rel.split(':')[1]
u_rel = rel.split('-')[0]
ca_rel = rel.split('-')[1]
Another difference from add_source() is that the function calls sys.exit(1)
if the configuration fails, whereas add_source() raises
SourceConfigurationError(). Another difference, is that add_source()
silently fails (with a juju_log command) if there is no matching source to
configure, whereas this function fails with a sys.exit(1)
if u_rel != ubuntu_rel:
e = 'Cannot install from Cloud Archive pocket %s on this Ubuntu '\
'version (%s)' % (ca_rel, ubuntu_rel)
error_out(e)
:param source: String_plus_key -- see above for details.
if 'staging' in ca_rel:
# staging is just a regular PPA.
os_rel = ca_rel.split('/')[0]
ppa = 'ppa:ubuntu-cloud-archive/%s-staging' % os_rel
cmd = 'add-apt-repository -y %s' % ppa
subprocess.check_call(cmd.split(' '))
return
Note that the behaviour on error is to log the error to the juju log and
then call sys.exit(1).
"""
# extract the key if there is one, denoted by a '|' in the rel
source, key = get_source_and_pgp_key(source_plus_key)
# map charm config options to actual archive pockets.
pockets = {
'folsom': 'precise-updates/folsom',
'folsom/updates': 'precise-updates/folsom',
'folsom/proposed': 'precise-proposed/folsom',
'grizzly': 'precise-updates/grizzly',
'grizzly/updates': 'precise-updates/grizzly',
'grizzly/proposed': 'precise-proposed/grizzly',
'havana': 'precise-updates/havana',
'havana/updates': 'precise-updates/havana',
'havana/proposed': 'precise-proposed/havana',
'icehouse': 'precise-updates/icehouse',
'icehouse/updates': 'precise-updates/icehouse',
'icehouse/proposed': 'precise-proposed/icehouse',
'juno': 'trusty-updates/juno',
'juno/updates': 'trusty-updates/juno',
'juno/proposed': 'trusty-proposed/juno',
'kilo': 'trusty-updates/kilo',
'kilo/updates': 'trusty-updates/kilo',
'kilo/proposed': 'trusty-proposed/kilo',
'liberty': 'trusty-updates/liberty',
'liberty/updates': 'trusty-updates/liberty',
'liberty/proposed': 'trusty-proposed/liberty',
'mitaka': 'trusty-updates/mitaka',
'mitaka/updates': 'trusty-updates/mitaka',
'mitaka/proposed': 'trusty-proposed/mitaka',
'newton': 'xenial-updates/newton',
'newton/updates': 'xenial-updates/newton',
'newton/proposed': 'xenial-proposed/newton',
'ocata': 'xenial-updates/ocata',
'ocata/updates': 'xenial-updates/ocata',
'ocata/proposed': 'xenial-proposed/ocata',
'pike': 'xenial-updates/pike',
'pike/updates': 'xenial-updates/pike',
'pike/proposed': 'xenial-proposed/pike',
'queens': 'xenial-updates/queens',
'queens/updates': 'xenial-updates/queens',
'queens/proposed': 'xenial-proposed/queens',
}
try:
pocket = pockets[ca_rel]
except KeyError:
e = 'Invalid Cloud Archive release specified: %s' % rel
error_out(e)
src = "deb %s %s main" % (CLOUD_ARCHIVE_URL, pocket)
apt_install('ubuntu-cloud-keyring', fatal=True)
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as f:
f.write(src)
else:
error_out("Invalid openstack-release specified: %s" % rel)
# handle the ordinary sources via add_source
try:
fetch_add_source(source, key, fail_invalid=True)
except SourceConfigError as se:
error_out(str(se))
def config_value_changed(option):
@ -677,7 +608,6 @@ def openstack_upgrade_available(package):
:returns: bool: : Returns True if configured installation source offers
a newer version of package.
"""
import apt_pkg as apt

View File

@ -191,6 +191,7 @@ def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d",
upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
sysv_file = os.path.join(initd_dir, service_name)
if init_is_systemd():
service('disable', service_name)
service('mask', service_name)
elif os.path.exists(upstart_file):
override_path = os.path.join(
@ -225,6 +226,7 @@ def service_resume(service_name, init_dir="/etc/init",
sysv_file = os.path.join(initd_dir, service_name)
if init_is_systemd():
service('unmask', service_name)
service('enable', service_name)
elif os.path.exists(upstart_file):
override_path = os.path.join(
init_dir, '{}.override'.format(service_name))

View File

@ -48,6 +48,13 @@ class AptLockError(Exception):
pass
class GPGKeyError(Exception):
"""Exception occurs when a GPG key cannot be fetched or used. The message
indicates what the problem is.
"""
pass
class BaseFetchHandler(object):
"""Base class for FetchHandler implementations in fetch plugins"""
@ -77,21 +84,22 @@ module = "charmhelpers.fetch.%s" % __platform__
fetch = importlib.import_module(module)
filter_installed_packages = fetch.filter_installed_packages
install = fetch.install
upgrade = fetch.upgrade
update = fetch.update
purge = fetch.purge
install = fetch.apt_install
upgrade = fetch.apt_upgrade
update = _fetch_update = fetch.apt_update
purge = fetch.apt_purge
add_source = fetch.add_source
if __platform__ == "ubuntu":
apt_cache = fetch.apt_cache
apt_install = fetch.install
apt_update = fetch.update
apt_upgrade = fetch.upgrade
apt_purge = fetch.purge
apt_install = fetch.apt_install
apt_update = fetch.apt_update
apt_upgrade = fetch.apt_upgrade
apt_purge = fetch.apt_purge
apt_mark = fetch.apt_mark
apt_hold = fetch.apt_hold
apt_unhold = fetch.apt_unhold
import_key = fetch.import_key
get_upstream_version = fetch.get_upstream_version
elif __platform__ == "centos":
yum_search = fetch.yum_search
@ -135,7 +143,7 @@ def configure_sources(update=False,
for source, key in zip(sources, keys):
add_source(source, key)
if update:
fetch.update(fatal=True)
_fetch_update(fatal=True)
def install_remote(source, *args, **kwargs):

View File

@ -132,7 +132,7 @@ def add_source(source, key=None):
key_file.write(key)
key_file.flush()
key_file.seek(0)
subprocess.check_call(['rpm', '--import', key_file])
subprocess.check_call(['rpm', '--import', key_file.name])
else:
subprocess.check_call(['rpm', '--import', key])

View File

@ -12,29 +12,47 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import OrderedDict
import os
import platform
import re
import six
import time
import subprocess
from tempfile import NamedTemporaryFile
from charmhelpers.core.host import (
lsb_release
)
from charmhelpers.core.hookenv import log
from charmhelpers.fetch import SourceConfigError
from charmhelpers.core.hookenv import (
log,
DEBUG,
)
from charmhelpers.fetch import SourceConfigError, GPGKeyError
PROPOSED_POCKET = (
"# Proposed\n"
"deb http://archive.ubuntu.com/ubuntu {}-proposed main universe "
"multiverse restricted\n")
PROPOSED_PORTS_POCKET = (
"# Proposed\n"
"deb http://ports.ubuntu.com/ubuntu-ports {}-proposed main universe "
"multiverse restricted\n")
# Only supports 64bit and ppc64 at the moment.
ARCH_TO_PROPOSED_POCKET = {
'x86_64': PROPOSED_POCKET,
'ppc64le': PROPOSED_PORTS_POCKET,
'aarch64': PROPOSED_PORTS_POCKET,
}
CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
"""
PROPOSED_POCKET = """# Proposed
deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
"""
CLOUD_ARCHIVE_POCKETS = {
# Folsom
'folsom': 'precise-updates/folsom',
'folsom/updates': 'precise-updates/folsom',
'precise-folsom': 'precise-updates/folsom',
'precise-folsom/updates': 'precise-updates/folsom',
'precise-updates/folsom': 'precise-updates/folsom',
@ -43,6 +61,7 @@ CLOUD_ARCHIVE_POCKETS = {
'precise-proposed/folsom': 'precise-proposed/folsom',
# Grizzly
'grizzly': 'precise-updates/grizzly',
'grizzly/updates': 'precise-updates/grizzly',
'precise-grizzly': 'precise-updates/grizzly',
'precise-grizzly/updates': 'precise-updates/grizzly',
'precise-updates/grizzly': 'precise-updates/grizzly',
@ -51,6 +70,7 @@ CLOUD_ARCHIVE_POCKETS = {
'precise-proposed/grizzly': 'precise-proposed/grizzly',
# Havana
'havana': 'precise-updates/havana',
'havana/updates': 'precise-updates/havana',
'precise-havana': 'precise-updates/havana',
'precise-havana/updates': 'precise-updates/havana',
'precise-updates/havana': 'precise-updates/havana',
@ -59,6 +79,7 @@ CLOUD_ARCHIVE_POCKETS = {
'precise-proposed/havana': 'precise-proposed/havana',
# Icehouse
'icehouse': 'precise-updates/icehouse',
'icehouse/updates': 'precise-updates/icehouse',
'precise-icehouse': 'precise-updates/icehouse',
'precise-icehouse/updates': 'precise-updates/icehouse',
'precise-updates/icehouse': 'precise-updates/icehouse',
@ -67,6 +88,7 @@ CLOUD_ARCHIVE_POCKETS = {
'precise-proposed/icehouse': 'precise-proposed/icehouse',
# Juno
'juno': 'trusty-updates/juno',
'juno/updates': 'trusty-updates/juno',
'trusty-juno': 'trusty-updates/juno',
'trusty-juno/updates': 'trusty-updates/juno',
'trusty-updates/juno': 'trusty-updates/juno',
@ -75,6 +97,7 @@ CLOUD_ARCHIVE_POCKETS = {
'trusty-proposed/juno': 'trusty-proposed/juno',
# Kilo
'kilo': 'trusty-updates/kilo',
'kilo/updates': 'trusty-updates/kilo',
'trusty-kilo': 'trusty-updates/kilo',
'trusty-kilo/updates': 'trusty-updates/kilo',
'trusty-updates/kilo': 'trusty-updates/kilo',
@ -83,6 +106,7 @@ CLOUD_ARCHIVE_POCKETS = {
'trusty-proposed/kilo': 'trusty-proposed/kilo',
# Liberty
'liberty': 'trusty-updates/liberty',
'liberty/updates': 'trusty-updates/liberty',
'trusty-liberty': 'trusty-updates/liberty',
'trusty-liberty/updates': 'trusty-updates/liberty',
'trusty-updates/liberty': 'trusty-updates/liberty',
@ -91,6 +115,7 @@ CLOUD_ARCHIVE_POCKETS = {
'trusty-proposed/liberty': 'trusty-proposed/liberty',
# Mitaka
'mitaka': 'trusty-updates/mitaka',
'mitaka/updates': 'trusty-updates/mitaka',
'trusty-mitaka': 'trusty-updates/mitaka',
'trusty-mitaka/updates': 'trusty-updates/mitaka',
'trusty-updates/mitaka': 'trusty-updates/mitaka',
@ -99,6 +124,7 @@ CLOUD_ARCHIVE_POCKETS = {
'trusty-proposed/mitaka': 'trusty-proposed/mitaka',
# Newton
'newton': 'xenial-updates/newton',
'newton/updates': 'xenial-updates/newton',
'xenial-newton': 'xenial-updates/newton',
'xenial-newton/updates': 'xenial-updates/newton',
'xenial-updates/newton': 'xenial-updates/newton',
@ -107,6 +133,7 @@ CLOUD_ARCHIVE_POCKETS = {
'xenial-proposed/newton': 'xenial-proposed/newton',
# Ocata
'ocata': 'xenial-updates/ocata',
'ocata/updates': 'xenial-updates/ocata',
'xenial-ocata': 'xenial-updates/ocata',
'xenial-ocata/updates': 'xenial-updates/ocata',
'xenial-updates/ocata': 'xenial-updates/ocata',
@ -131,6 +158,7 @@ CLOUD_ARCHIVE_POCKETS = {
'xenial-queens/newton': 'xenial-proposed/queens',
}
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries.
CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times.
@ -161,7 +189,7 @@ def apt_cache(in_memory=True, progress=None):
return apt_pkg.Cache(progress)
def install(packages, options=None, fatal=False):
def apt_install(packages, options=None, fatal=False):
"""Install one or more packages."""
if options is None:
options = ['--option=Dpkg::Options::=--force-confold']
@ -178,7 +206,7 @@ def install(packages, options=None, fatal=False):
_run_apt_command(cmd, fatal)
def upgrade(options=None, fatal=False, dist=False):
def apt_upgrade(options=None, fatal=False, dist=False):
"""Upgrade all packages."""
if options is None:
options = ['--option=Dpkg::Options::=--force-confold']
@ -193,13 +221,13 @@ def upgrade(options=None, fatal=False, dist=False):
_run_apt_command(cmd, fatal)
def update(fatal=False):
def apt_update(fatal=False):
"""Update local apt cache."""
cmd = ['apt-get', 'update']
_run_apt_command(cmd, fatal)
def purge(packages, fatal=False):
def apt_purge(packages, fatal=False):
"""Purge one or more packages."""
cmd = ['apt-get', '--assume-yes', 'purge']
if isinstance(packages, six.string_types):
@ -233,7 +261,45 @@ def apt_unhold(packages, fatal=False):
return apt_mark(packages, 'unhold', fatal=fatal)
def add_source(source, key=None):
def import_key(keyid):
"""Import a key in either ASCII Armor or Radix64 format.
`keyid` is either the keyid to fetch from a PGP server, or
the key in ASCII armor foramt.
:param keyid: String of key (or key id).
:raises: GPGKeyError if the key could not be imported
"""
key = keyid.strip()
if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and
key.endswith('-----END PGP PUBLIC KEY BLOCK-----')):
log("PGP key found (looks like ASCII Armor format)", level=DEBUG)
log("Importing ASCII Armor PGP key", level=DEBUG)
with NamedTemporaryFile() as keyfile:
with open(keyfile.name, 'w') as fd:
fd.write(key)
fd.write("\n")
cmd = ['apt-key', 'add', keyfile.name]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
error = "Error importing PGP key '{}'".format(key)
log(error)
raise GPGKeyError(error)
else:
log("PGP key found (looks like Radix64 format)", level=DEBUG)
log("Importing PGP key from keyserver", level=DEBUG)
cmd = ['apt-key', 'adv', '--keyserver',
'hkp://keyserver.ubuntu.com:80', '--recv-keys', key]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
error = "Error importing PGP key '{}'".format(key)
log(error)
raise GPGKeyError(error)
def add_source(source, key=None, fail_invalid=False):
"""Add a package source to this system.
@param source: a URL or sources.list entry, as supported by
@ -249,6 +315,33 @@ def add_source(source, key=None):
such as 'cloud:icehouse'
'distro' may be used as a noop
Full list of source specifications supported by the function are:
'distro': A NOP; i.e. it has no effect.
'proposed': the proposed deb spec [2] is wrtten to
/etc/apt/sources.list/proposed
'distro-proposed': adds <version>-proposed to the debs [2]
'ppa:<ppa-name>': add-apt-repository --yes <ppa_name>
'deb <deb-spec>': add-apt-repository --yes deb <deb-spec>
'http://....': add-apt-repository --yes http://...
'cloud-archive:<spec>': add-apt-repository -yes cloud-archive:<spec>
'cloud:<release>[-staging]': specify a Cloud Archive pocket <release> with
optional staging version. If staging is used then the staging PPA [2]
with be used. If staging is NOT used then the cloud archive [3] will be
added, and the 'ubuntu-cloud-keyring' package will be added for the
current distro.
Otherwise the source is not recognised and this is logged to the juju log.
However, no error is raised, unless sys_error_on_exit is True.
[1] deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
where {} is replaced with the derived pocket name.
[2] deb http://archive.ubuntu.com/ubuntu {}-proposed \
main universe multiverse restricted
where {} is replaced with the lsb_release codename (e.g. xenial)
[3] deb http://ubuntu-cloud.archive.canonical.com/ubuntu <pocket>
to /etc/apt/sources.list.d/cloud-archive-list
@param key: A key to be added to the system's APT keyring and used
to verify the signatures on packages. Ideally, this should be an
ASCII format GPG public key including the block headers. A GPG key
@ -256,51 +349,141 @@ def add_source(source, key=None):
available to retrieve the actual public key from a public keyserver
placing your Juju environment at risk. ppa and cloud archive keys
are securely added automtically, so sould not be provided.
@param fail_invalid: (boolean) if True, then the function raises a
SourceConfigError is there is no matching installation source.
@raises SourceConfigError() if for cloud:<pocket>, the <pocket> is not a
valid pocket in CLOUD_ARCHIVE_POCKETS
"""
_mapping = OrderedDict([
(r"^distro$", lambda: None), # This is a NOP
(r"^(?:proposed|distro-proposed)$", _add_proposed),
(r"^cloud-archive:(.*)$", _add_apt_repository),
(r"^((?:deb |http:|https:|ppa:).*)$", _add_apt_repository),
(r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging),
(r"^cloud:(.*)-(.*)$", _add_cloud_distro_check),
(r"^cloud:(.*)$", _add_cloud_pocket),
])
if source is None:
log('Source is not present. Skipping')
return
if (source.startswith('ppa:') or
source.startswith('http') or
source.startswith('deb ') or
source.startswith('cloud-archive:')):
cmd = ['add-apt-repository', '--yes', source]
_run_with_retries(cmd)
elif source.startswith('cloud:'):
install(filter_installed_packages(['ubuntu-cloud-keyring']),
fatal=True)
pocket = source.split(':')[-1]
if pocket not in CLOUD_ARCHIVE_POCKETS:
raise SourceConfigError(
'Unsupported cloud: source option %s' %
pocket)
actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
apt.write(CLOUD_ARCHIVE.format(actual_pocket))
elif source == 'proposed':
release = lsb_release()['DISTRIB_CODENAME']
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
apt.write(PROPOSED_POCKET.format(release))
elif source == 'distro':
pass
source = ''
for r, fn in six.iteritems(_mapping):
m = re.match(r, source)
if m:
# call the assoicated function with the captured groups
# raises SourceConfigError on error.
fn(*m.groups())
if key:
try:
import_key(key)
except GPGKeyError as e:
raise SourceConfigError(str(e))
break
else:
log("Unknown source: {!r}".format(source))
# nothing matched. log an error and maybe sys.exit
err = "Unknown source: {!r}".format(source)
log(err)
if fail_invalid:
raise SourceConfigError(err)
if key:
if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
with NamedTemporaryFile('w+') as key_file:
key_file.write(key)
key_file.flush()
key_file.seek(0)
subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
else:
# Note that hkp: is in no way a secure protocol. Using a
# GPG key id is pointless from a security POV unless you
# absolutely trust your network and DNS.
subprocess.check_call(['apt-key', 'adv', '--keyserver',
'hkp://keyserver.ubuntu.com:80', '--recv',
key])
def _add_proposed():
"""Add the PROPOSED_POCKET as /etc/apt/source.list.d/proposed.list
Uses lsb_release()['DISTRIB_CODENAME'] to determine the correct staza for
the deb line.
For intel architecutres PROPOSED_POCKET is used for the release, but for
other architectures PROPOSED_PORTS_POCKET is used for the release.
"""
release = lsb_release()['DISTRIB_CODENAME']
arch = platform.machine()
if arch not in six.iterkeys(ARCH_TO_PROPOSED_POCKET):
raise SourceConfigError("Arch {} not supported for (distro-)proposed"
.format(arch))
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
apt.write(ARCH_TO_PROPOSED_POCKET[arch].format(release))
def _add_apt_repository(spec):
"""Add the spec using add_apt_repository
:param spec: the parameter to pass to add_apt_repository
"""
_run_with_retries(['add-apt-repository', '--yes', spec])
def _add_cloud_pocket(pocket):
"""Add a cloud pocket as /etc/apt/sources.d/cloud-archive.list
Note that this overwrites the existing file if there is one.
This function also converts the simple pocket in to the actual pocket using
the CLOUD_ARCHIVE_POCKETS mapping.
:param pocket: string representing the pocket to add a deb spec for.
:raises: SourceConfigError if the cloud pocket doesn't exist or the
requested release doesn't match the current distro version.
"""
apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
fatal=True)
if pocket not in CLOUD_ARCHIVE_POCKETS:
raise SourceConfigError(
'Unsupported cloud: source option %s' %
pocket)
actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
apt.write(CLOUD_ARCHIVE.format(actual_pocket))
def _add_cloud_staging(cloud_archive_release, openstack_release):
"""Add the cloud staging repository which is in
ppa:ubuntu-cloud-archive/<openstack_release>-staging
This function checks that the cloud_archive_release matches the current
codename for the distro that charm is being installed on.
:param cloud_archive_release: string, codename for the release.
:param openstack_release: String, codename for the openstack release.
:raises: SourceConfigError if the cloud_archive_release doesn't match the
current version of the os.
"""
_verify_is_ubuntu_rel(cloud_archive_release, openstack_release)
ppa = 'ppa:ubuntu-cloud-archive/{}-staging'.format(openstack_release)
cmd = 'add-apt-repository -y {}'.format(ppa)
_run_with_retries(cmd.split(' '))
def _add_cloud_distro_check(cloud_archive_release, openstack_release):
"""Add the cloud pocket, but also check the cloud_archive_release against
the current distro, and use the openstack_release as the full lookup.
This just calls _add_cloud_pocket() with the openstack_release as pocket
to get the correct cloud-archive.list for dpkg to work with.
:param cloud_archive_release:String, codename for the distro release.
:param openstack_release: String, spec for the release to look up in the
CLOUD_ARCHIVE_POCKETS
:raises: SourceConfigError if this is the wrong distro, or the pocket spec
doesn't exist.
"""
_verify_is_ubuntu_rel(cloud_archive_release, openstack_release)
_add_cloud_pocket("{}-{}".format(cloud_archive_release, openstack_release))
def _verify_is_ubuntu_rel(release, os_release):
"""Verify that the release is in the same as the current ubuntu release.
:param release: String, lowercase for the release.
:param os_release: String, the os_release being asked for
:raises: SourceConfigError if the release is not the same as the ubuntu
release.
"""
ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
if release != ubuntu_rel:
raise SourceConfigError(
'Invalid Cloud Archive release specified: {}-{} on this Ubuntu'
'version ({})'.format(release, os_release, ubuntu_rel))
def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
@ -316,9 +499,12 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
:param: cmd_env: dict: Environment variables to add to the command run.
"""
env = os.environ.copy()
env = None
kwargs = {}
if cmd_env:
env = os.environ.copy()
env.update(cmd_env)
kwargs['env'] = env
if not retry_message:
retry_message = "Failed executing '{}'".format(" ".join(cmd))
@ -330,7 +516,8 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
retry_results = (None,) + retry_exitcodes
while result in retry_results:
try:
result = subprocess.check_call(cmd, env=env)
# result = subprocess.check_call(cmd, env=env)
result = subprocess.check_call(cmd, **kwargs)
except subprocess.CalledProcessError as e:
retry_count = retry_count + 1
if retry_count > max_retries:
@ -343,6 +530,7 @@ def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
def _run_apt_command(cmd, fatal=False):
"""Run an apt command with optional retries.
:param: cmd: str: The apt command to run.
:param: fatal: bool: Whether the command's output should be checked and
retried.
"""

View File

@ -14,6 +14,11 @@
# Bootstrap charm-helpers, installing its dependencies if necessary using
# only standard libraries.
from __future__ import print_function
from __future__ import absolute_import
import functools
import inspect
import subprocess
import sys
@ -34,3 +39,59 @@ except ImportError:
else:
subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
import yaml # flake8: noqa
# Holds a list of mapping of mangled function names that have been deprecated
# using the @deprecate decorator below. This is so that the warning is only
# printed once for each usage of the function.
__deprecated_functions = {}
def deprecate(warning, date=None, log=None):
"""Add a deprecation warning the first time the function is used.
The date, which is a string in semi-ISO8660 format indicate the year-month
that the function is officially going to be removed.
usage:
@deprecate('use core/fetch/add_source() instead', '2017-04')
def contributed_add_source_thing(...):
...
And it then prints to the log ONCE that the function is deprecated.
The reason for passing the logging function (log) is so that hookenv.log
can be used for a charm if needed.
:param warning: String to indicat where it has moved ot.
:param date: optional sting, in YYYY-MM format to indicate when the
function will definitely (probably) be removed.
:param log: The log function to call to log. If not, logs to stdout
"""
def wrap(f):
@functools.wraps(f)
def wrapped_f(*args, **kwargs):
try:
module = inspect.getmodule(f)
file = inspect.getsourcefile(f)
lines = inspect.getsourcelines(f)
f_name = "{}-{}-{}..{}-{}".format(
module.__name__, file, lines[0], lines[-1], f.__name__)
except (IOError, TypeError):
# assume it was local, so just use the name of the function
f_name = f.__name__
if f_name not in __deprecated_functions:
__deprecated_functions[f_name] = True
s = "DEPRECATION WARNING: Function {} is being removed".format(
f.__name__)
if date:
s = "{} on/around {}".format(s, date)
if warning:
s = "{} : {}".format(s, warning)
if log:
log(s)
else:
print(s)
return f(*args, **kwargs)
return wrapped_f
return wrap

View File

@ -191,6 +191,7 @@ def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d",
upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
sysv_file = os.path.join(initd_dir, service_name)
if init_is_systemd():
service('disable', service_name)
service('mask', service_name)
elif os.path.exists(upstart_file):
override_path = os.path.join(
@ -225,6 +226,7 @@ def service_resume(service_name, init_dir="/etc/init",
sysv_file = os.path.join(initd_dir, service_name)
if init_is_systemd():
service('unmask', service_name)
service('enable', service_name)
elif os.path.exists(upstart_file):
override_path = os.path.join(
init_dir, '{}.override'.format(service_name))