openstack
/
charm-neutron-gateway
Code Issues Proposed changes

[gnuoy,r=james-page,t=james-page] Add support for specifying external port by mac address 

Also resync charm-helpers
This commit is contained in:
James Page 2014-03-14 13:24:39 +00:00
parent fcc2f15d3c 9757d38a9b
commit 7faeded915
14 changed files with 303 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@ -54,6 +54,22 @@ See upstream [Neutron multi extnet](http://docs.openstack.org/trunk/config-refer
Configuration Options
---------------------
External Port Configuration
===========================
If the port to be used for external traffic is consistent accross all physical
servers then is can be specified by simply setting ext-port to the nic id:
quantum-gateway:
ext-port: eth2
However, if it varies between hosts then the mac addresses of the external
nics for each host can be passed as a space seperated list:
quantum-gateway:
ext-port: <MAC ext port host 1> <MAC ext port host 2> <MAC ext port host 3>
Multiple Floating Pools
=======================

8
hooks/charmhelpers/contrib/hahelpers/cluster.py
View File

@ -126,17 +126,17 @@ def determine_api_port(public_port):
return public_port - (i * 10)
def determine_haproxy_port(public_port):
def determine_apache_port(public_port):
'''
Description: Determine correct proxy listening port based on public IP +
existence of HTTPS reverse proxy.
Description: Determine correct apache listening port based on public IP +
state of the cluster.
public_port: int: standard public port for given service
returns: int: the correct listening port for the HAProxy service
'''
i = 0
if https():
if len(peer_units()) > 0 or is_clustered():
i += 1
return public_port - (i * 10)

137
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -23,15 +23,13 @@ from charmhelpers.core.hookenv import (
unit_get,
unit_private_ip,
ERROR,
WARNING,
)
from charmhelpers.contrib.hahelpers.cluster import (
determine_apache_port,
determine_api_port,
determine_haproxy_port,
https,
is_clustered,
peer_units,
is_clustered
)
from charmhelpers.contrib.hahelpers.apache import (
@ -68,6 +66,43 @@ def context_complete(ctxt):
return True
def config_flags_parser(config_flags):
if config_flags.find('==') >= 0:
log("config_flags is not in expected format (key=value)",
level=ERROR)
raise OSContextError
# strip the following from each value.
post_strippers = ' ,'
# we strip any leading/trailing '=' or ' ' from the string then
# split on '='.
split = config_flags.strip(' =').split('=')
limit = len(split)
flags = {}
for i in xrange(0, limit - 1):
current = split[i]
next = split[i + 1]
vindex = next.rfind(',')
if (i == limit - 2) or (vindex < 0):
value = next
else:
value = next[:vindex]
if i == 0:
key = current
else:
# if this not the first entry, expect an embedded key.
index = current.rfind(',')
if index < 0:
log("invalid config value(s) at index %s" % (i),
level=ERROR)
raise OSContextError
key = current[index + 1:]
# Add to collection.
flags[key.strip(post_strippers)] = value.rstrip(post_strippers)
return flags
class OSContextGenerator(object):
interfaces = []
@ -182,10 +217,17 @@ class AMQPContext(OSContextGenerator):
# Sufficient information found = break out!
break
# Used for active/active rabbitmq >= grizzly
ctxt['rabbitmq_hosts'] = []
for unit in related_units(rid):
ctxt['rabbitmq_hosts'].append(relation_get('private-address',
rid=rid, unit=unit))
if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \
len(related_units(rid)) > 1:
if relation_get('ha_queues'):
ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues')
else:
ctxt['rabbitmq_ha_queues'] = False
rabbitmq_hosts = []
for unit in related_units(rid):
rabbitmq_hosts.append(relation_get('private-address',
rid=rid, unit=unit))
ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts)
if not context_complete(ctxt):
return {}
else:
@ -199,10 +241,13 @@ class CephContext(OSContextGenerator):
'''This generates context for /etc/ceph/ceph.conf templates'''
if not relation_ids('ceph'):
return {}
log('Generating template context for ceph')
mon_hosts = []
auth = None
key = None
use_syslog = str(config('use-syslog')).lower()
for rid in relation_ids('ceph'):
for unit in related_units(rid):
mon_hosts.append(relation_get('private-address', rid=rid,
@ -214,6 +259,7 @@ class CephContext(OSContextGenerator):
'mon_hosts': ' '.join(mon_hosts),
'auth': auth,
'key': key,
'use_syslog': use_syslog
}
if not os.path.isdir('/etc/ceph'):
@ -286,6 +332,7 @@ class ImageServiceContext(OSContextGenerator):
class ApacheSSLContext(OSContextGenerator):
"""
Generates a context for an apache vhost configuration that configures
HTTPS reverse proxying for one or many endpoints. Generated context
@ -341,17 +388,15 @@ class ApacheSSLContext(OSContextGenerator):
'private_address': unit_get('private-address'),
'endpoints': []
}
for ext_port in self.external_ports:
if peer_units() or is_clustered():
int_port = determine_haproxy_port(ext_port)
else:
int_port = determine_api_port(ext_port)
for api_port in self.external_ports:
ext_port = determine_apache_port(api_port)
int_port = determine_api_port(api_port)
portmap = (int(ext_port), int(int_port))
ctxt['endpoints'].append(portmap)
return ctxt
class NeutronContext(object):
class NeutronContext(OSContextGenerator):
interfaces = []
@property
@ -412,6 +457,22 @@ class NeutronContext(object):
return nvp_ctxt
def neutron_ctxt(self):
if https():
proto = 'https'
else:
proto = 'http'
if is_clustered():
host = config('vip')
else:
host = unit_get('private-address')
url = '%s://%s:%s' % (proto, host, '9696')
ctxt = {
'network_manager': self.network_manager,
'neutron_url': url,
}
return ctxt
def __call__(self):
self._ensure_packages()
@ -421,40 +482,44 @@ class NeutronContext(object):
if not self.plugin:
return {}
ctxt = {'network_manager': self.network_manager}
ctxt = self.neutron_ctxt()
if self.plugin == 'ovs':
ctxt.update(self.ovs_ctxt())
elif self.plugin == 'nvp':
ctxt.update(self.nvp_ctxt())
alchemy_flags = config('neutron-alchemy-flags')
if alchemy_flags:
flags = config_flags_parser(alchemy_flags)
ctxt['neutron_alchemy_flags'] = flags
self._save_flag_file()
return ctxt
class OSConfigFlagContext(OSContextGenerator):
'''
Responsible adding user-defined config-flags in charm config to a
to a template context.
'''
"""
Responsible for adding user-defined config-flags in charm config to a
template context.
NOTE: the value of config-flags may be a comma-separated list of
key=value pairs and some Openstack config files support
comma-separated lists as values.
"""
def __call__(self):
config_flags = config('config-flags')
if not config_flags or config_flags in ['None', '']:
if not config_flags:
return {}
config_flags = config_flags.split(',')
flags = {}
for flag in config_flags:
if '=' not in flag:
log('Improperly formatted config-flag, expected k=v '
'got %s' % flag, level=WARNING)
continue
k, v = flag.split('=')
flags[k.strip()] = v
ctxt = {'user_config_flags': flags}
return ctxt
flags = config_flags_parser(config_flags)
return {'user_config_flags': flags}
class SubordinateConfigContext(OSContextGenerator):
"""
Responsible for inspecting relations to subordinates that
may be exporting required config via a json blob.
@ -495,6 +560,7 @@ class SubordinateConfigContext(OSContextGenerator):
}
"""
def __init__(self, service, config_file, interface):
"""
:param service : Service name key to query in any subordinate
@ -539,3 +605,12 @@ class SubordinateConfigContext(OSContextGenerator):
ctxt['sections'] = {}
return ctxt
class SyslogContext(OSContextGenerator):
def __call__(self):
ctxt = {
'use_syslog': config('use-syslog')
}
return ctxt

20
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -41,6 +41,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
('quantal', 'folsom'),
('raring', 'grizzly'),
('saucy', 'havana'),
('trusty', 'icehouse')
])
@ -201,7 +202,7 @@ def os_release(package, base='essex'):
def import_key(keyid):
cmd = "apt-key adv --keyserver keyserver.ubuntu.com " \
cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 " \
"--recv-keys %s" % keyid
try:
subprocess.check_call(cmd.split(' '))
@ -260,6 +261,9 @@ def configure_installation_source(rel):
'havana': 'precise-updates/havana',
'havana/updates': 'precise-updates/havana',
'havana/proposed': 'precise-proposed/havana',
'icehouse': 'precise-updates/icehouse',
'icehouse/updates': 'precise-updates/icehouse',
'icehouse/proposed': 'precise-proposed/icehouse',
}
try:
@ -411,7 +415,7 @@ def get_host_ip(hostname):
return ns_query(hostname)
def get_hostname(address):
def get_hostname(address, fqdn=True):
"""
Resolves hostname for given IP, or returns the input
if it is already a hostname.
@ -430,7 +434,11 @@ def get_hostname(address):
if not result:
return None
# strip trailing .
if result.endswith('.'):
return result[:-1]
return result
if fqdn:
# strip trailing .
if result.endswith('.'):
return result[:-1]
else:
return result
else:
return result.split('.')[0]

8
hooks/charmhelpers/contrib/storage/linux/ceph.py
View File

@ -49,6 +49,9 @@ CEPH_CONF = """[global]
auth supported = {auth}
keyring = {keyring}
mon host = {mon_hosts}
log to syslog = {use_syslog}
err to syslog = {use_syslog}
clog to syslog = {use_syslog}
"""
@ -194,7 +197,7 @@ def get_ceph_nodes():
return hosts
def configure(service, key, auth):
def configure(service, key, auth, use_syslog):
''' Perform basic configuration of Ceph '''
create_keyring(service, key)
create_key_file(service, key)
@ -202,7 +205,8 @@ def configure(service, key, auth):
with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
ceph_conf.write(CEPH_CONF.format(auth=auth,
keyring=_keyring_path(service),
mon_hosts=",".join(map(str, hosts))))
mon_hosts=",".join(map(str, hosts)),
use_syslog=use_syslog))
modprobe('rbd')

2
hooks/charmhelpers/contrib/storage/linux/utils.py
View File

@ -22,4 +22,4 @@ def zap_disk(block_device):
:param block_device: str: Full path of block device to clean.
'''
check_call(['sgdisk', '--zap-all', block_device])
check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device])

6
hooks/charmhelpers/core/hookenv.py
View File

@ -8,6 +8,7 @@ import os
import json
import yaml
import subprocess
import sys
import UserDict
from subprocess import CalledProcessError
@ -149,6 +150,11 @@ def service_name():
return local_unit().split('/')[0]
def hook_name():
"""The name of the currently executing hook"""
return os.path.basename(sys.argv[0])
@cached
def config(scope=None):
"""Juju charm configuration"""

56
hooks/charmhelpers/core/host.py
View File

@ -194,7 +194,7 @@ def file_hash(path):
return None
def restart_on_change(restart_map):
def restart_on_change(restart_map, stopstart=False):
"""Restart services based on configuration files changing
This function is used a decorator, for example
@ -219,8 +219,14 @@ def restart_on_change(restart_map):
for path in restart_map:
if checksums[path] != file_hash(path):
restarts += restart_map[path]
for service_name in list(OrderedDict.fromkeys(restarts)):
service('restart', service_name)
services_list = list(OrderedDict.fromkeys(restarts))
if not stopstart:
for service_name in services_list:
service('restart', service_name)
else:
for action in ['stop', 'start']:
for service_name in services_list:
service(action, service_name)
return wrapped_f
return wrap
@ -245,3 +251,47 @@ def pwgen(length=None):
random_chars = [
random.choice(alphanumeric_chars) for _ in range(length)]
return(''.join(random_chars))
def list_nics(nic_type):
'''Return a list of nics of given type(s)'''
if isinstance(nic_type, basestring):
int_types = [nic_type]
else:
int_types = nic_type
interfaces = []
for int_type in int_types:
cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
ip_output = subprocess.check_output(cmd).split('\n')
ip_output = (line for line in ip_output if line)
for line in ip_output:
if line.split()[1].startswith(int_type):
interfaces.append(line.split()[1].replace(":", ""))
return interfaces
def set_nic_mtu(nic, mtu):
'''Set MTU on a network interface'''
cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
subprocess.check_call(cmd)
def get_nic_mtu(nic):
cmd = ['ip', 'addr', 'show', nic]
ip_output = subprocess.check_output(cmd).split('\n')
mtu = ""
for line in ip_output:
words = line.split()
if 'mtu' in words:
mtu = words[words.index("mtu") + 1]
return mtu
def get_nic_hwaddr(nic):
cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
ip_output = subprocess.check_output(cmd)
hwaddr = ""
words = ip_output.split()
if 'link/ether' in words:
hwaddr = words[words.index('link/ether') + 1]
return hwaddr

52
hooks/charmhelpers/fetch/__init__.py
View File

@ -13,6 +13,7 @@ from charmhelpers.core.hookenv import (
log,
)
import apt_pkg
import os
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
@ -43,8 +44,16 @@ CLOUD_ARCHIVE_POCKETS = {
'precise-havana/updates': 'precise-updates/havana',
'precise-updates/havana': 'precise-updates/havana',
'havana/proposed': 'precise-proposed/havana',
'precies-havana/proposed': 'precise-proposed/havana',
'precise-havana/proposed': 'precise-proposed/havana',
'precise-proposed/havana': 'precise-proposed/havana',
# Icehouse
'icehouse': 'precise-updates/icehouse',
'precise-icehouse': 'precise-updates/icehouse',
'precise-icehouse/updates': 'precise-updates/icehouse',
'precise-updates/icehouse': 'precise-updates/icehouse',
'icehouse/proposed': 'precise-proposed/icehouse',
'precise-icehouse/proposed': 'precise-proposed/icehouse',
'precise-proposed/icehouse': 'precise-proposed/icehouse',
}
@ -66,8 +75,10 @@ def filter_installed_packages(packages):
def apt_install(packages, options=None, fatal=False):
"""Install one or more packages"""
options = options or []
cmd = ['apt-get', '-y']
if options is None:
options = ['--option=Dpkg::Options::=--force-confold']
cmd = ['apt-get', '--assume-yes']
cmd.extend(options)
cmd.append('install')
if isinstance(packages, basestring):
@ -76,10 +87,14 @@ def apt_install(packages, options=None, fatal=False):
cmd.extend(packages)
log("Installing {} with options: {}".format(packages,
options))
env = os.environ.copy()
if 'DEBIAN_FRONTEND' not in env:
env['DEBIAN_FRONTEND'] = 'noninteractive'
if fatal:
subprocess.check_call(cmd)
subprocess.check_call(cmd, env=env)
else:
subprocess.call(cmd)
subprocess.call(cmd, env=env)
def apt_update(fatal=False):
@ -93,7 +108,7 @@ def apt_update(fatal=False):
def apt_purge(packages, fatal=False):
"""Purge one or more packages"""
cmd = ['apt-get', '-y', 'purge']
cmd = ['apt-get', '--assume-yes', 'purge']
if isinstance(packages, basestring):
cmd.append(packages)
else:
@ -120,17 +135,23 @@ def apt_hold(packages, fatal=False):
def add_source(source, key=None):
if source is None:
log('Source is not present. Skipping')
return
if (source.startswith('ppa:') or
source.startswith('http:') or
source.startswith('http') or
source.startswith('deb ') or
source.startswith('cloud-archive:')):
source.startswith('cloud-archive:')):
subprocess.check_call(['add-apt-repository', '--yes', source])
elif source.startswith('cloud:'):
apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
fatal=True)
pocket = source.split(':')[-1]
if pocket not in CLOUD_ARCHIVE_POCKETS:
raise SourceConfigError('Unsupported cloud: source option %s' % pocket)
raise SourceConfigError(
'Unsupported cloud: source option %s' %
pocket)
actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
apt.write(CLOUD_ARCHIVE.format(actual_pocket))
@ -139,7 +160,9 @@ def add_source(source, key=None):
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
apt.write(PROPOSED_POCKET.format(release))
if key:
subprocess.check_call(['apt-key', 'import', key])
subprocess.check_call(['apt-key', 'adv', '--keyserver',
'keyserver.ubuntu.com', '--recv',
key])
class SourceConfigError(Exception):
@ -220,7 +243,9 @@ def install_from_config(config_var_name):
class BaseFetchHandler(object):
"""Base class for FetchHandler implementations in fetch plugins"""
def can_handle(self, source):
"""Returns True if the source can be handled. Otherwise returns
a string explaining why it cannot"""
@ -248,10 +273,13 @@ def plugins(fetch_handlers=None):
for handler_name in fetch_handlers:
package, classname = handler_name.rsplit('.', 1)
try:
handler_class = getattr(importlib.import_module(package), classname)
handler_class = getattr(
importlib.import_module(package),
classname)
plugin_list.append(handler_class())
except (ImportError, AttributeError):
# Skip missing plugins so that they can be ommitted from
# installation if desired
log("FetchHandler {} not found, skipping plugin".format(handler_name))
log("FetchHandler {} not found, skipping plugin".format(
handler_name))
return plugin_list

21
hooks/quantum_contexts.py
View File

@ -2,6 +2,10 @@
import os
import uuid
import socket
from charmhelpers.core.host import (
list_nics,
get_nic_hwaddr
)
from charmhelpers.core.hookenv import (
config,
relation_ids,
@ -23,6 +27,7 @@ from charmhelpers.contrib.openstack.utils import (
from charmhelpers.contrib.hahelpers.cluster import(
eligible_leader
)
import re
DB_USER = "quantum"
QUANTUM_DB = "quantum"
@ -121,10 +126,20 @@ class L3AgentContext(OSContextGenerator):
class ExternalPortContext(OSContextGenerator):
def __call__(self):
if config('ext-port'):
return {"ext_port": config('ext-port')}
else:
if not config('ext-port'):
return None
hwaddrs = {}
for nic in list_nics(['eth', 'bond']):
hwaddrs[get_nic_hwaddr(nic)] = nic
mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I)
for entry in config('ext-port').split():
entry = entry.strip()
if re.match(mac_regex, entry):
if entry in hwaddrs:
return {"ext_port": hwaddrs[entry]}
else:
return {"ext_port": entry}
return None
class QuantumGatewayContext(OSContextGenerator):

6
hooks/quantum_utils.py
View File

@ -434,6 +434,6 @@ def configure_ovs():
full_restart()
add_bridge(INT_BRIDGE)
add_bridge(EXT_BRIDGE)
ext_port = config('ext-port')
if ext_port:
add_bridge_port(EXT_BRIDGE, ext_port)
ext_port_ctx = ExternalPortContext()()
if ext_port_ctx is not None and ext_port_ctx['ext_port']:
add_bridge_port(EXT_BRIDGE, ext_port_ctx['ext_port'])

2
revision
View File

@ -1 +1 @@
60
63

25
unit_tests/test_quantum_contexts.py
View File

@ -16,6 +16,8 @@ TO_PATCH = [
'apt_install',
'get_os_codename_install_source',
'eligible_leader',
'list_nics',
'get_nic_hwaddr',
]
@ -136,11 +138,32 @@ class TestExternalPortContext(CharmTestCase):
self.assertEquals(quantum_contexts.ExternalPortContext()(),
None)
def test_ext_port(self):
def test_ext_port_eth(self):
self.config.return_value = 'eth1010'
self.assertEquals(quantum_contexts.ExternalPortContext()(),
{'ext_port': 'eth1010'})
def test_ext_port_mac(self):
machine_macs = {
'eth0': 'fe:c5:ce:8e:2b:00',
'eth1': 'fe:c5:ce:8e:2b:01',
'eth2': 'fe:c5:ce:8e:2b:02',
'eth3': 'fe:c5:ce:8e:2b:03',
}
absent_macs = "aa:a5:ae:ae:ab:a4 "
config_macs = absent_macs + " " + machine_macs['eth2']
def get_hwaddr(arg):
return machine_macs[arg]
self.config.return_value = config_macs
self.list_nics.return_value = machine_macs.keys()
self.get_nic_hwaddr.side_effect = get_hwaddr
self.assertEquals(quantum_contexts.ExternalPortContext()(),
{'ext_port': 'eth2'})
self.config.return_value = absent_macs
self.assertEquals(quantum_contexts.ExternalPortContext()(),
None)
class TestL3AgentContext(CharmTestCase):
def setUp(self):

11
unit_tests/test_quantum_utils.py
View File

@ -34,6 +34,7 @@ TO_PATCH = [
'full_restart',
'service_running',
'NetworkServiceContext',
'ExternalPortContext',
'unit_private_ip',
'relations_of_type',
'service_stop',
@ -100,6 +101,8 @@ class TestQuantumUtils(CharmTestCase):
self.config.side_effect = self.test_config.get
self.test_config.set('plugin', 'ovs')
self.test_config.set('ext-port', 'eth0')
self.ExternalPortContext.return_value = \
DummyExternalPortContext(return_value={'ext_port': 'eth0'})
quantum_utils.configure_ovs()
self.add_bridge.assert_has_calls([
call('br-int'),
@ -264,6 +267,14 @@ class DummyNetworkServiceContext():
def __call__(self):
return self.return_value
class DummyExternalPortContext():
def __init__(self, return_value):
self.return_value = return_value
def __call__(self):
return self.return_value
agents_all_alive = {
'DHCP Agent': {
'agents': [