538 lines
18 KiB
Python
Executable File
538 lines
18 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
import os
|
|
import sys
|
|
|
|
from subprocess import (
|
|
check_call,
|
|
CalledProcessError,
|
|
)
|
|
|
|
from lib.swift_utils import (
|
|
SwiftProxyCharmException,
|
|
register_configs,
|
|
restart_map,
|
|
services,
|
|
determine_packages,
|
|
ensure_swift_dir,
|
|
SWIFT_RINGS,
|
|
get_www_dir,
|
|
initialize_ring,
|
|
SWIFT_HA_RES,
|
|
get_zone,
|
|
do_openstack_upgrade,
|
|
setup_ipv6,
|
|
update_rings,
|
|
balance_rings,
|
|
fully_synced,
|
|
sync_proxy_rings,
|
|
broadcast_rings_available,
|
|
mark_www_rings_deleted,
|
|
SwiftProxyClusterRPC,
|
|
get_first_available_value,
|
|
all_responses_equal,
|
|
ensure_www_dir_permissions,
|
|
is_paused,
|
|
pause_aware_restart_on_change,
|
|
assess_status,
|
|
)
|
|
|
|
import charmhelpers.contrib.openstack.utils as openstack
|
|
from charmhelpers.contrib.hahelpers.cluster import (
|
|
is_elected_leader,
|
|
)
|
|
from charmhelpers.core.hookenv import (
|
|
config,
|
|
local_unit,
|
|
remote_unit,
|
|
unit_get,
|
|
relation_set,
|
|
relation_ids,
|
|
relation_get,
|
|
related_units,
|
|
log,
|
|
DEBUG,
|
|
INFO,
|
|
WARNING,
|
|
ERROR,
|
|
Hooks, UnregisteredHookError,
|
|
open_port,
|
|
status_set,
|
|
)
|
|
from charmhelpers.core.host import (
|
|
service_reload,
|
|
service_restart,
|
|
service_stop,
|
|
service_start,
|
|
)
|
|
from charmhelpers.fetch import (
|
|
apt_install,
|
|
apt_update,
|
|
)
|
|
from charmhelpers.payload.execd import execd_preinstall
|
|
from charmhelpers.contrib.openstack.ip import (
|
|
canonical_url,
|
|
PUBLIC,
|
|
INTERNAL,
|
|
ADMIN,
|
|
)
|
|
from charmhelpers.contrib.network.ip import (
|
|
get_iface_for_address,
|
|
get_netmask_for_address,
|
|
get_address_in_network,
|
|
get_ipv6_addr,
|
|
is_ipv6,
|
|
format_ipv6_addr,
|
|
)
|
|
from charmhelpers.contrib.openstack.context import ADDRESS_TYPES
|
|
from charmhelpers.contrib.charmsupport import nrpe
|
|
|
|
extra_pkgs = [
|
|
"haproxy",
|
|
"python-jinja2"
|
|
]
|
|
|
|
hooks = Hooks()
|
|
CONFIGS = register_configs()
|
|
|
|
|
|
@hooks.hook('install.real')
|
|
def install():
|
|
status_set('maintenance', 'Executing pre-install')
|
|
execd_preinstall()
|
|
src = config('openstack-origin')
|
|
if src != 'distro':
|
|
openstack.configure_installation_source(src)
|
|
|
|
status_set('maintenance', 'Installing apt packages')
|
|
apt_update(fatal=True)
|
|
rel = openstack.get_os_codename_install_source(src)
|
|
pkgs = determine_packages(rel)
|
|
apt_install(pkgs, fatal=True)
|
|
apt_install(extra_pkgs, fatal=True)
|
|
ensure_swift_dir()
|
|
# configure a directory on webserver for distributing rings.
|
|
ensure_www_dir_permissions(get_www_dir())
|
|
|
|
|
|
@hooks.hook('config-changed')
|
|
@pause_aware_restart_on_change(restart_map())
|
|
def config_changed():
|
|
if is_elected_leader(SWIFT_HA_RES):
|
|
log("Leader established, generating ring builders", level=INFO)
|
|
# initialize new storage rings.
|
|
for path in SWIFT_RINGS.itervalues():
|
|
if not os.path.exists(path):
|
|
initialize_ring(path,
|
|
config('partition-power'),
|
|
config('replicas'),
|
|
config('min-hours'))
|
|
|
|
if config('prefer-ipv6'):
|
|
status_set('maintenance', 'Configuring ipv6')
|
|
setup_ipv6()
|
|
|
|
configure_https()
|
|
open_port(config('bind-port'))
|
|
update_nrpe_config()
|
|
|
|
# Determine whether or not we should do an upgrade.
|
|
if not config('action-managed-upgrade') and \
|
|
openstack.openstack_upgrade_available('python-swift'):
|
|
do_openstack_upgrade(CONFIGS)
|
|
status_set('maintenance', 'Running openstack upgrade')
|
|
|
|
status_set('maintenance', 'Updating and balancing rings')
|
|
update_rings(min_part_hours=config('min-hours'))
|
|
|
|
if not config('disable-ring-balance') and is_elected_leader(SWIFT_HA_RES):
|
|
# Try ring balance. If rings are balanced, no sync will occur.
|
|
balance_rings()
|
|
|
|
for r_id in relation_ids('identity-service'):
|
|
keystone_joined(relid=r_id)
|
|
|
|
|
|
@hooks.hook('identity-service-relation-joined')
|
|
def keystone_joined(relid=None):
|
|
port = config('bind-port')
|
|
admin_url = '%s:%s' % (canonical_url(CONFIGS, ADMIN), port)
|
|
internal_url = ('%s:%s/v1/AUTH_$(tenant_id)s' %
|
|
(canonical_url(CONFIGS, INTERNAL), port))
|
|
public_url = ('%s:%s/v1/AUTH_$(tenant_id)s' %
|
|
(canonical_url(CONFIGS, PUBLIC), port))
|
|
region = config('region')
|
|
roles = config('operator-roles')
|
|
|
|
relation_set(service='swift', region=region, public_url=public_url,
|
|
internal_url=internal_url, admin_url=admin_url,
|
|
requested_roles=roles, relation_id=relid)
|
|
|
|
|
|
@hooks.hook('identity-service-relation-changed')
|
|
@pause_aware_restart_on_change(restart_map())
|
|
def keystone_changed():
|
|
configure_https()
|
|
|
|
|
|
@hooks.hook('swift-storage-relation-joined')
|
|
def storage_joined():
|
|
if not is_elected_leader(SWIFT_HA_RES):
|
|
log("New storage relation joined - stopping proxy until ring builder "
|
|
"synced", level=INFO)
|
|
service_stop('swift-proxy')
|
|
|
|
# This unit is not currently responsible for distributing rings but
|
|
# may become so at some time in the future so we do this to avoid the
|
|
# possibility of storage nodes getting out-of-date rings by deprecating
|
|
# any existing ones from the www dir.
|
|
mark_www_rings_deleted()
|
|
|
|
|
|
@hooks.hook('swift-storage-relation-changed')
|
|
@pause_aware_restart_on_change(restart_map())
|
|
def storage_changed():
|
|
"""Storage relation.
|
|
|
|
Only the leader unit can update and distribute rings so if we are not the
|
|
leader we ignore this event and wait for a resync request from the leader.
|
|
"""
|
|
if not is_elected_leader(SWIFT_HA_RES):
|
|
log("Not the leader - ignoring storage relation until leader ready.",
|
|
level=DEBUG)
|
|
return
|
|
|
|
log("Leader established, updating ring builders", level=INFO)
|
|
addr = relation_get('private-address')
|
|
if config('prefer-ipv6'):
|
|
host_ip = format_ipv6_addr(addr)
|
|
if not host_ip:
|
|
errmsg = ("Did not get IPv6 address from storage relation "
|
|
"(got=%s)" % (addr))
|
|
raise SwiftProxyCharmException(errmsg)
|
|
else:
|
|
host_ip = openstack.get_host_ip(addr)
|
|
|
|
zone = get_zone(config('zone-assignment'))
|
|
node_settings = {
|
|
'ip': host_ip,
|
|
'zone': zone,
|
|
'account_port': relation_get('account_port'),
|
|
'object_port': relation_get('object_port'),
|
|
'container_port': relation_get('container_port'),
|
|
}
|
|
|
|
if None in node_settings.itervalues():
|
|
missing = [k for k, v in node_settings.iteritems() if v is None]
|
|
log("Relation not ready - some required values not provided by "
|
|
"relation (missing=%s)" % (', '.join(missing)), level=INFO)
|
|
return None
|
|
|
|
for k in ['zone', 'account_port', 'object_port', 'container_port']:
|
|
node_settings[k] = int(node_settings[k])
|
|
|
|
CONFIGS.write_all()
|
|
|
|
# Allow for multiple devs per unit, passed along as a : separated list
|
|
# Update and balance rings.
|
|
nodes = []
|
|
devs = relation_get('device')
|
|
if devs:
|
|
for dev in devs.split(':'):
|
|
node = {k: v for k, v in node_settings.items()}
|
|
node['device'] = dev
|
|
nodes.append(node)
|
|
|
|
update_rings(nodes)
|
|
if not is_paused():
|
|
# Restart proxy here in case no config changes made (so
|
|
# pause_aware_restart_on_change() ineffective).
|
|
service_restart('swift-proxy')
|
|
|
|
|
|
@hooks.hook('swift-storage-relation-broken')
|
|
@pause_aware_restart_on_change(restart_map())
|
|
def storage_broken():
|
|
CONFIGS.write_all()
|
|
|
|
|
|
@hooks.hook('cluster-relation-joined')
|
|
def cluster_joined(relation_id=None):
|
|
for addr_type in ADDRESS_TYPES:
|
|
netaddr_cfg = 'os-{}-network'.format(addr_type)
|
|
address = get_address_in_network(config(netaddr_cfg))
|
|
if address:
|
|
settings = {'{}-address'.format(addr_type): address}
|
|
relation_set(relation_id=relation_id, relation_settings=settings)
|
|
|
|
if config('prefer-ipv6'):
|
|
private_addr = get_ipv6_addr(exc_list=[config('vip')])[0]
|
|
relation_set(relation_id=relation_id,
|
|
relation_settings={'private-address': private_addr})
|
|
else:
|
|
private_addr = unit_get('private-address')
|
|
|
|
|
|
def all_peers_stopped(responses):
|
|
"""Establish whether all peers have stopped their proxy services.
|
|
|
|
Each peer unit will set stop-proxy-service-ack to rq value to indicate that
|
|
it has stopped its proxy service. We wait for all units to be stopped
|
|
before triggering a sync. Peer services will be restarted once their rings
|
|
are synced with the leader.
|
|
|
|
To be safe, default expectation is that api is still running.
|
|
"""
|
|
rq_key = SwiftProxyClusterRPC.KEY_STOP_PROXY_SVC
|
|
ack_key = SwiftProxyClusterRPC.KEY_STOP_PROXY_SVC_ACK
|
|
token = relation_get(attribute=rq_key, unit=local_unit())
|
|
if not token or token != responses[0].get(ack_key):
|
|
log("Unmatched token in ack (expected=%s, got=%s)" %
|
|
(token, responses[0].get(ack_key)), level=DEBUG)
|
|
return False
|
|
|
|
if not all_responses_equal(responses, ack_key):
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def cluster_leader_actions():
|
|
"""Cluster relation hook actions to be performed by leader units.
|
|
|
|
NOTE: must be called by leader from cluster relation hook.
|
|
"""
|
|
log("Cluster changed by unit=%s (local is leader)" % (remote_unit()),
|
|
level=DEBUG)
|
|
|
|
# If we have received an ack, check other units
|
|
settings = relation_get() or {}
|
|
ack_key = SwiftProxyClusterRPC.KEY_STOP_PROXY_SVC_ACK
|
|
|
|
# Protect against leader changing mid-sync
|
|
if settings.get(SwiftProxyClusterRPC.KEY_STOP_PROXY_SVC):
|
|
log("Sync request received yet this is leader unit. This would "
|
|
"indicate that the leader has changed mid-sync - stopping proxy "
|
|
"and notifying peers", level=ERROR)
|
|
service_stop('swift-proxy')
|
|
SwiftProxyClusterRPC().notify_leader_changed()
|
|
return
|
|
elif ack_key in settings:
|
|
token = settings[ack_key]
|
|
# Find out if all peer units have been stopped.
|
|
responses = []
|
|
for rid in relation_ids('cluster'):
|
|
for unit in related_units(rid):
|
|
responses.append(relation_get(rid=rid, unit=unit))
|
|
|
|
# Ensure all peers stopped before starting sync
|
|
if all_peers_stopped(responses):
|
|
key = 'peers-only'
|
|
if not all_responses_equal(responses, key, must_exist=False):
|
|
msg = ("Did not get equal response from every peer unit for "
|
|
"'%s'" % (key))
|
|
raise SwiftProxyCharmException(msg)
|
|
|
|
peers_only = int(get_first_available_value(responses, key,
|
|
default=0))
|
|
log("Syncing rings and builders (peers-only=%s)" % (peers_only),
|
|
level=DEBUG)
|
|
broadcast_rings_available(token, storage=not peers_only)
|
|
else:
|
|
log("Not all peer apis stopped - skipping sync until all peers "
|
|
"ready (got %s)" % (responses), level=INFO)
|
|
|
|
CONFIGS.write_all()
|
|
|
|
|
|
def cluster_non_leader_actions():
|
|
"""Cluster relation hook actions to be performed by non-leader units.
|
|
|
|
NOTE: must be called by non-leader from cluster relation hook.
|
|
"""
|
|
log("Cluster changed by unit=%s (local is non-leader)" % (remote_unit()),
|
|
level=DEBUG)
|
|
settings = relation_get() or {}
|
|
|
|
# Check whether we have been requested to stop proxy service
|
|
rq_key = SwiftProxyClusterRPC.KEY_STOP_PROXY_SVC
|
|
token = settings.get(rq_key, None)
|
|
if token:
|
|
log("Peer request to stop proxy service received (%s) - sending ack" %
|
|
(token), level=INFO)
|
|
service_stop('swift-proxy')
|
|
peers_only = settings.get('peers-only', None)
|
|
rq = SwiftProxyClusterRPC().stop_proxy_ack(echo_token=token,
|
|
echo_peers_only=peers_only)
|
|
relation_set(relation_settings=rq)
|
|
return
|
|
|
|
# Check if there are any builder files we can sync from the leader.
|
|
log("Non-leader peer - checking if updated rings available", level=DEBUG)
|
|
broker = settings.get('builder-broker', None)
|
|
if not broker:
|
|
log("No update available", level=DEBUG)
|
|
if not is_paused():
|
|
service_start('swift-proxy')
|
|
return
|
|
|
|
builders_only = int(settings.get('sync-only-builders', 0))
|
|
path = os.path.basename(get_www_dir())
|
|
try:
|
|
sync_proxy_rings('http://%s/%s' % (broker, path),
|
|
rings=not builders_only)
|
|
except CalledProcessError:
|
|
log("Ring builder sync failed, builders not yet available - "
|
|
"leader not ready?", level=WARNING)
|
|
return None
|
|
|
|
# Re-enable the proxy once all builders and rings are synced
|
|
if fully_synced():
|
|
log("Ring builders synced - starting proxy", level=INFO)
|
|
CONFIGS.write_all()
|
|
if not is_paused():
|
|
service_start('swift-proxy')
|
|
else:
|
|
log("Not all builders and rings synced yet - waiting for peer sync "
|
|
"before starting proxy", level=INFO)
|
|
|
|
|
|
@hooks.hook('cluster-relation-changed')
|
|
@pause_aware_restart_on_change(restart_map())
|
|
def cluster_changed():
|
|
key = SwiftProxyClusterRPC.KEY_NOTIFY_LEADER_CHANGED
|
|
leader_changed = relation_get(attribute=key)
|
|
if leader_changed:
|
|
log("Leader changed notification received from peer unit. Since this "
|
|
"most likely occurred during a ring sync proxies will be "
|
|
"disabled until the leader is restored and a fresh sync request "
|
|
"is set out", level=WARNING)
|
|
service_stop("swift-proxy")
|
|
return
|
|
|
|
if is_elected_leader(SWIFT_HA_RES):
|
|
cluster_leader_actions()
|
|
else:
|
|
cluster_non_leader_actions()
|
|
|
|
|
|
@hooks.hook('ha-relation-changed')
|
|
def ha_relation_changed():
|
|
clustered = relation_get('clustered')
|
|
if clustered:
|
|
log("Cluster configured, notifying other services and updating "
|
|
"keystone endpoint configuration", level=INFO)
|
|
# Tell all related services to start using
|
|
# the VIP instead
|
|
for r_id in relation_ids('identity-service'):
|
|
keystone_joined(relid=r_id)
|
|
|
|
|
|
@hooks.hook('ha-relation-joined')
|
|
def ha_relation_joined():
|
|
# Obtain the config values necessary for the cluster config. These
|
|
# include multicast port and interface to bind to.
|
|
corosync_bindiface = config('ha-bindiface')
|
|
corosync_mcastport = config('ha-mcastport')
|
|
vip = config('vip')
|
|
if not vip:
|
|
msg = 'Unable to configure hacluster as vip not provided'
|
|
raise SwiftProxyCharmException(msg)
|
|
|
|
# Obtain resources
|
|
resources = {'res_swift_haproxy': 'lsb:haproxy'}
|
|
resource_params = {'res_swift_haproxy': 'op monitor interval="5s"'}
|
|
|
|
vip_group = []
|
|
for vip in vip.split():
|
|
if is_ipv6(vip):
|
|
res_swift_vip = 'ocf:heartbeat:IPv6addr'
|
|
vip_params = 'ipv6addr'
|
|
else:
|
|
res_swift_vip = 'ocf:heartbeat:IPaddr2'
|
|
vip_params = 'ip'
|
|
|
|
iface = get_iface_for_address(vip)
|
|
if iface is not None:
|
|
vip_key = 'res_swift_{}_vip'.format(iface)
|
|
resources[vip_key] = res_swift_vip
|
|
resource_params[vip_key] = (
|
|
'params {ip}="{vip}" cidr_netmask="{netmask}"'
|
|
' nic="{iface}"'.format(ip=vip_params,
|
|
vip=vip,
|
|
iface=iface,
|
|
netmask=get_netmask_for_address(vip))
|
|
)
|
|
vip_group.append(vip_key)
|
|
|
|
if len(vip_group) >= 1:
|
|
relation_set(groups={'grp_swift_vips': ' '.join(vip_group)})
|
|
|
|
init_services = {'res_swift_haproxy': 'haproxy'}
|
|
clones = {'cl_swift_haproxy': 'res_swift_haproxy'}
|
|
|
|
relation_set(init_services=init_services,
|
|
corosync_bindiface=corosync_bindiface,
|
|
corosync_mcastport=corosync_mcastport,
|
|
resources=resources,
|
|
resource_params=resource_params,
|
|
clones=clones)
|
|
|
|
|
|
def configure_https():
|
|
"""Enables SSL API Apache config if appropriate and kicks identity-service
|
|
with any required api updates.
|
|
"""
|
|
# need to write all to ensure changes to the entire request pipeline
|
|
# propagate (c-api, haprxy, apache)
|
|
CONFIGS.write_all()
|
|
if 'https' in CONFIGS.complete_contexts():
|
|
cmd = ['a2ensite', 'openstack_https_frontend']
|
|
check_call(cmd)
|
|
else:
|
|
cmd = ['a2dissite', 'openstack_https_frontend']
|
|
check_call(cmd)
|
|
|
|
# Apache 2.4 required enablement of configuration
|
|
if os.path.exists('/usr/sbin/a2enconf'):
|
|
check_call(['a2enconf', 'swift-rings'])
|
|
|
|
if not is_paused():
|
|
# TODO: improve this by checking if local CN certs are available
|
|
# first then checking reload status (see LP #1433114).
|
|
service_reload('apache2', restart_on_failure=True)
|
|
|
|
for rid in relation_ids('identity-service'):
|
|
keystone_joined(relid=rid)
|
|
|
|
env_vars = {'OPENSTACK_SERVICE_SWIFT': 'proxy-server',
|
|
'OPENSTACK_PORT_API': config('bind-port'),
|
|
'OPENSTACK_PORT_MEMCACHED': 11211}
|
|
openstack.save_script_rc(**env_vars)
|
|
|
|
|
|
@hooks.hook('nrpe-external-master-relation-joined',
|
|
'nrpe-external-master-relation-changed')
|
|
def update_nrpe_config():
|
|
# python-dbus is used by check_upstart_job
|
|
apt_install('python-dbus')
|
|
hostname = nrpe.get_nagios_hostname()
|
|
current_unit = nrpe.get_nagios_unit_name()
|
|
nrpe_setup = nrpe.NRPE(hostname=hostname)
|
|
nrpe.copy_nrpe_checks()
|
|
nrpe.add_init_service_checks(nrpe_setup, services(), current_unit)
|
|
nrpe.add_haproxy_checks(nrpe_setup, current_unit)
|
|
nrpe_setup.write()
|
|
|
|
|
|
def main():
|
|
try:
|
|
hooks.execute(sys.argv)
|
|
except UnregisteredHookError as e:
|
|
log('Unknown hook {} - skipping.'.format(e), level=DEBUG)
|
|
assess_status(CONFIGS)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|