From 242549117f7af9c5b23028de579b145a65db493f Mon Sep 17 00:00:00 2001 From: Bilal Baqar Date: Sat, 21 Nov 2015 19:26:23 -0800 Subject: [PATCH 1/6] Adding fabric/data network support --- config.yaml | 20 ++++++- hooks/pg_gw_context.py | 7 ++- hooks/pg_gw_hooks.py | 11 ++++ hooks/pg_gw_utils.py | 95 +++++++++++++++++++++++++------- templates/kilo/ifcs.conf | 2 +- unit_tests/test_pg_gw_context.py | 7 ++- unit_tests/test_pg_gw_hooks.py | 15 +---- 7 files changed, 119 insertions(+), 38 deletions(-) diff --git a/config.yaml b/config.yaml index 8b2e69c..07e7676 100644 --- a/config.yaml +++ b/config.yaml @@ -13,10 +13,28 @@ options: type: string default: 'juju-br0' description: The interface connected to PLUMgrid Managment network. + os-data-network: + type: string + default: + description: | + The IP address and netmask of the OpenStack Data network (e.g., + 192.168.0.0/24) + . + This network will be used for tenant network traffic in overlay + networks. + fabric-interfaces: + default: 'MANAGEMENT' + type: string + description: | + Interfaces that will provide fabric connectivity on the gateway nodes. + Provided in form of json in a string. These interfaces have to be connected + to the os-data-network specified in the config. Default value is MANAGEMENT which + will configure the management interface as the fabric interface on each + director. network-device-mtu: type: string default: '1580' - description: The MTU size for interfaces managed by director. + description: The MTU size for interfaces managed by gateway. install_sources: default: 'ppa:plumgrid-team/stable' type: string diff --git a/hooks/pg_gw_context.py b/hooks/pg_gw_context.py index d94f3df..f3dada7 100644 --- a/hooks/pg_gw_context.py +++ b/hooks/pg_gw_context.py @@ -71,8 +71,13 @@ class PGGwContext(context.NeutronContext): pg_ctxt['local_ip'] = pg_dir_ips unit_hostname = get_unit_hostname() pg_ctxt['pg_hostname'] = unit_hostname - from pg_gw_utils import get_mgmt_interface, get_gw_interfaces + from pg_gw_utils import ( + get_mgmt_interface, + get_gw_interfaces, + get_fabric_interface + ) pg_ctxt['interface'] = get_mgmt_interface() + pg_ctxt['fabric_interface'] = get_fabric_interface() pg_ctxt['label'] = unit_hostname pg_ctxt['fabric_mode'] = 'host' pg_ctxt['ext_interfaces'] = get_gw_interfaces() diff --git a/hooks/pg_gw_hooks.py b/hooks/pg_gw_hooks.py index 8aa54d3..d976b7e 100755 --- a/hooks/pg_gw_hooks.py +++ b/hooks/pg_gw_hooks.py @@ -11,6 +11,7 @@ from charmhelpers.core.hookenv import ( Hooks, UnregisteredHookError, log, + config, ) from charmhelpers.fetch import ( @@ -29,6 +30,7 @@ from pg_gw_utils import ( remove_iovisor, ensure_mtu, add_lcm_key, + fabric_interface_changed ) hooks = Hooks() @@ -73,6 +75,15 @@ def config_changed(): if add_lcm_key(): log("PLUMgrid LCM Key added") return 1 + charm_config = config() + if charm_config.changed('fabric-interfaces'): + if not fabric_interface_changed(): + log("Fabric interface already set") + return 1 + if charm_config.changed('os-data-network'): + if charm_config['fabric-interfaces'] == 'MANAGEMENT': + log('Fabric running on managment network') + return 1 stop_pg() configure_sources(update=True) pkgs = determine_packages() diff --git a/hooks/pg_gw_utils.py b/hooks/pg_gw_utils.py index d84cfa3..e3586d0 100644 --- a/hooks/pg_gw_utils.py +++ b/hooks/pg_gw_utils.py @@ -13,6 +13,8 @@ from charmhelpers.contrib.network.ip import ( get_iface_from_addr, get_bridges, get_bridge_nics, + is_address_in_network, + get_iface_addr ) from charmhelpers.core.host import ( write_file, @@ -170,22 +172,23 @@ def remove_iovisor(): time.sleep(1) +def interface_exists(interface): + ''' + Checks if interface exists on node. + ''' + try: + subprocess.check_call(['ip', 'link', 'show', interface], + stdout=open(os.devnull, 'w'), + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + return False + return True + + def get_mgmt_interface(): ''' Returns the managment interface. ''' - def interface_exists(interface): - ''' - Checks if interface exists on node. - ''' - try: - subprocess.check_call(['ip', 'link', 'show', interface], - stdout=open(os.devnull, 'w'), - stderr=subprocess.STDOUT) - except subprocess.CalledProcessError: - return False - return True - mgmt_interface = config('mgmt-interface') if interface_exists(mgmt_interface): return mgmt_interface @@ -195,20 +198,74 @@ def get_mgmt_interface(): return get_iface_from_addr(unit_get('private-address')) +def fabric_interface_changed(): + ''' + Returns true if interface for node changed. + ''' + fabric_interface = get_fabric_interface() + try: + with open(PG_IFCS_CONF, 'r') as ifcs: + for line in ifcs: + if 'fabric_core' in line: + if line.split()[0] == fabric_interface: + return False + except IOError: + return True + return True + + +def get_fabric_interface(): + ''' + Returns the fabric interface. + ''' + fabric_interfaces = config('fabric-interfaces') + if fabric_interfaces == 'MANAGEMENT': + return get_mgmt_interface() + else: + try: + all_fabric_interfaces = json.loads(fabric_interfaces) + except ValueError: + raise ValueError('Invalid json provided for fabric interfaces') + hostname = get_unit_hostname() + if hostname in all_fabric_interfaces: + node_fabric_interface = all_fabric_interfaces[hostname] + elif 'DEFAULT' in all_fabric_interfaces: + node_fabric_interface = all_fabric_interfaces['DEFAULT'] + else: + raise ValueError('No fabric interface provided for node') + if interface_exists(node_fabric_interface): + if is_address_in_network(config('os-data-network'), + get_iface_addr(node_fabric_interface)[0]): + return node_fabric_interface + else: + raise ValueError('Fabric interface not in fabric network') + else: + log('Provided fabric interface %s does not exist' + % node_fabric_interface) + raise ValueError('Provided fabric interface does not exist') + return node_fabric_interface + + def get_gw_interfaces(): ''' Gateway node can have multiple interfaces. This function parses json provided in config to get all gateway interfaces for this node. ''' - node_interfaces = ['eth1'] + node_interfaces = [] try: all_interfaces = json.loads(config('external-interfaces')) except ValueError: - log("Invalid JSON") - return node_interfaces + raise ValueError("Invalid json provided for gateway interfaces") hostname = get_unit_hostname() if hostname in all_interfaces: node_interfaces = all_interfaces[hostname].split(',') + elif 'DEFAULT' in all_interfaces: + node_interfaces = all_interfaces['DEFAULT'].split(',') + for interface in node_interfaces: + if not interface_exists(interface): + log('Provided gateway interface %s does not exist' + % interface) + raise ValueError('Provided gateway interface does not exist') return node_interfaces @@ -217,12 +274,12 @@ def ensure_mtu(): Ensures required MTU of the underlying networking of the node. ''' interface_mtu = config('network-device-mtu') - mgmt_interface = get_mgmt_interface() - if mgmt_interface in get_bridges(): - attached_interfaces = get_bridge_nics(mgmt_interface) + fabric_interface = get_fabric_interface() + if fabric_interface in get_bridges(): + attached_interfaces = get_bridge_nics(fabric_interface) for interface in attached_interfaces: set_nic_mtu(interface, interface_mtu) - set_nic_mtu(mgmt_interface, interface_mtu) + set_nic_mtu(fabric_interface, interface_mtu) def _exec_cmd(cmd=None, error_msg='Command exited with ERRORs', fatal=False): diff --git a/templates/kilo/ifcs.conf b/templates/kilo/ifcs.conf index 40166fe..657fbd5 100644 --- a/templates/kilo/ifcs.conf +++ b/templates/kilo/ifcs.conf @@ -1,4 +1,4 @@ -{{ interface }} = fabric_core host +{{ fabric_interface }} = fabric_core host {% if ext_interfaces -%} {% for ip in ext_interfaces -%} {{ ip }} = access_phys diff --git a/unit_tests/test_pg_gw_context.py b/unit_tests/test_pg_gw_context.py index 6604f1f..18d9f98 100644 --- a/unit_tests/test_pg_gw_context.py +++ b/unit_tests/test_pg_gw_context.py @@ -38,9 +38,10 @@ class PGGwContextTest(CharmTestCase): @patch.object(charmhelpers.contrib.openstack.context, 'neutron_plugin_attribute') @patch.object(utils, 'get_mgmt_interface') + @patch.object(utils, 'get_fabric_interface') @patch.object(utils, 'get_gw_interfaces') - def test_neutroncc_context_api_rel(self, _gw_int, _mgmt_int, - _npa, _pg_dir_settings, + def test_neutroncc_context_api_rel(self, _gw_int, _fabric_int, + _mgmt_int, _npa, _pg_dir_settings, _save_flag_file, _config_flag, _unit_get, _unit_priv_ip, _config, _is_clus, _https, _ens_pkgs): @@ -59,6 +60,7 @@ class PGGwContextTest(CharmTestCase): _config_flag.return_value = False _pg_dir_settings.return_value = {'pg_dir_ip': '192.168.100.201'} _mgmt_int.return_value = 'juju-br0' + _fabric_int.return_value = 'juju-br0' _gw_int.return_value = ['eth1'] napi_ctxt = context.PGGwContext() expect = { @@ -72,6 +74,7 @@ class PGGwContextTest(CharmTestCase): 'neutron_url': 'https://192.168.100.201:9696', 'pg_hostname': 'node0', 'interface': 'juju-br0', + 'fabric_interface': 'juju-br0', 'label': 'node0', 'fabric_mode': 'host', 'neutron_alchemy_flags': False, diff --git a/unit_tests/test_pg_gw_hooks.py b/unit_tests/test_pg_gw_hooks.py index bebdb9d..6bbf5f6 100644 --- a/unit_tests/test_pg_gw_hooks.py +++ b/unit_tests/test_pg_gw_hooks.py @@ -69,21 +69,8 @@ class PGGwHooksTests(CharmTestCase): self.restart_pg.assert_called_with() def test_config_changed_hook(self): - _pkgs = ['plumgrid-lxc', 'iovisor-dkms'] - self.add_lcm_key.return_value = 0 - self.determine_packages.return_value = [_pkgs] + self.add_lcm_key.return_value = 1 self._call_hook('config-changed') - self.stop_pg.assert_called_with() - self.configure_sources.assert_called_with(update=True) - self.apt_install.assert_has_calls([ - call(_pkgs, fatal=True, - options=['--force-yes']), - ]) - self.load_iovisor.assert_called_with() - self.ensure_mtu.assert_called_with() - self.ensure_files.assert_called_with() - self.CONFIGS.write_all.assert_called_with() - self.restart_pg.assert_called_with() def test_stop(self): _pkgs = ['plumgrid-lxc', 'iovisor-dkms'] From 806d2da3e7d673eca9504e4ee77bf953cefd9c69 Mon Sep 17 00:00:00 2001 From: Bilal Baqar Date: Thu, 3 Mar 2016 13:49:53 -0800 Subject: [PATCH 3/6] Fixing lint --- Makefile | 2 +- unit_tests/test_pg_gw_hooks.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c89fd2f..b4bfdbf 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ virtualenv: netaddr jinja2 lint: virtualenv - .venv/bin/flake8 --exclude hooks/charmhelpers hooks unit_tests tests + .venv/bin/flake8 --exclude hooks/charmhelpers hooks unit_tests tests --ignore E402 @charm proof unit_test: virtualenv diff --git a/unit_tests/test_pg_gw_hooks.py b/unit_tests/test_pg_gw_hooks.py index 6bbf5f6..8e87dc8 100644 --- a/unit_tests/test_pg_gw_hooks.py +++ b/unit_tests/test_pg_gw_hooks.py @@ -1,5 +1,6 @@ from mock import MagicMock, patch, call from test_utils import CharmTestCase + with patch('charmhelpers.core.hookenv.config') as config: config.return_value = 'neutron' import pg_gw_utils as utils From eb3a56478657f92478dcf5aab1905689bc29a2f6 Mon Sep 17 00:00:00 2001 From: Bilal Baqar Date: Wed, 9 Mar 2016 12:29:15 -0800 Subject: [PATCH 4/6] Fix for CFB-829 --- hooks/pg_gw_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/pg_gw_context.py b/hooks/pg_gw_context.py index f3dada7..b2bc7b5 100644 --- a/hooks/pg_gw_context.py +++ b/hooks/pg_gw_context.py @@ -60,7 +60,7 @@ class PGGwContext(context.NeutronContext): return {} pg_dir_ips = '' - pg_dir_settings = _pg_dir_settings() + pg_dir_settings = sorted(_pg_dir_settings()) single_ip = True for ip in pg_dir_settings: if single_ip: From bf8557803512b4ffd2002e71b575fc470a01b596 Mon Sep 17 00:00:00 2001 From: Bilal Baqar Date: Thu, 10 Mar 2016 14:39:19 -0800 Subject: [PATCH 5/6] Loading Specific Iptables in install hook --- hooks/pg_gw_hooks.py | 4 ++- hooks/pg_gw_utils.py | 50 ++++++++++++++++++++++++++++++++-- unit_tests/test_pg_gw_hooks.py | 1 + 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/hooks/pg_gw_hooks.py b/hooks/pg_gw_hooks.py index d976b7e..a64030b 100755 --- a/hooks/pg_gw_hooks.py +++ b/hooks/pg_gw_hooks.py @@ -30,7 +30,8 @@ from pg_gw_utils import ( remove_iovisor, ensure_mtu, add_lcm_key, - fabric_interface_changed + fabric_interface_changed, + load_iptables, ) hooks = Hooks() @@ -42,6 +43,7 @@ def install(): ''' Install hook is run when the charm is first deployed on a node. ''' + load_iptables() configure_sources(update=True) pkgs = determine_packages() for pkg in pkgs: diff --git a/hooks/pg_gw_utils.py b/hooks/pg_gw_utils.py index e3586d0..47577de 100644 --- a/hooks/pg_gw_utils.py +++ b/hooks/pg_gw_utils.py @@ -22,7 +22,8 @@ from charmhelpers.core.host import ( service_stop, ) from charmhelpers.fetch import ( - apt_cache + apt_cache, + apt_install ) from charmhelpers.contrib.storage.linux.ceph import modprobe from charmhelpers.core.host import set_nic_mtu @@ -143,7 +144,6 @@ def restart_pg(): ''' service_stop('plumgrid') time.sleep(30) - _exec_cmd(cmd=['iptables', '-F']) service_start('plumgrid') time.sleep(30) @@ -327,3 +327,49 @@ def add_lcm_key(): fa.write('\n') fa.close() return 1 + + +def load_iptables(): + network = get_cidr_from_iface(get_mgmt_interface()) + if network: + _exec_cmd(['sudo', 'iptables', '-A', 'INPUT', '-p', 'tcp', + '-j', 'ACCEPT', '-s', network, '-d', + network, '-m', 'state', '--state', 'NEW']) + _exec_cmd(['sudo', 'iptables', '-A', 'INPUT', '-p', 'udp', '-j', + 'ACCEPT', '-s', network, '-d', network, + '-m', 'state', '--state', 'NEW']) + _exec_cmd(['sudo', 'iptables', '-I', 'INPUT', '-s', network, + '-d', '224.0.0.18/32', '-j', 'ACCEPT']) + _exec_cmd(['sudo', 'iptables', '-I', 'INPUT', '-p', 'vrrp', '-j', + 'ACCEPT']) + _exec_cmd(['sudo', 'iptables', '-A', 'INPUT', '-p', 'tcp', '-j', + 'ACCEPT', '-d', config('plumgrid-virtual-ip'), '-m', + 'state', '--state', 'NEW']) + apt_install('iptables-persistent') + + +def get_cidr_from_iface(interface): + if not interface: + return None + apt_install('ohai') + try: + os_info = subprocess.check_output(['ohai', '-l', 'fatal']) + except OSError: + log('Unable to get operating system information') + return None + try: + os_info_json = json.loads(os_info) + except ValueError: + log('Unable to determine network') + return None + device = os_info_json['network']['interfaces'].get(interface) + if device is not None: + if device.get('routes'): + routes = device['routes'] + for net in routes: + if 'scope' in net: + return net.get('destination') + else: + return None + else: + return None diff --git a/unit_tests/test_pg_gw_hooks.py b/unit_tests/test_pg_gw_hooks.py index 8e87dc8..a1b5779 100644 --- a/unit_tests/test_pg_gw_hooks.py +++ b/unit_tests/test_pg_gw_hooks.py @@ -30,6 +30,7 @@ TO_PATCH = [ 'ensure_mtu', 'add_lcm_key', 'determine_packages', + 'load_iptables' ] NEUTRON_CONF_DIR = "/etc/neutron" From 395eaafb5c00f0236f9c42696d528f13ec77d646 Mon Sep 17 00:00:00 2001 From: Bilal Baqar Date: Fri, 11 Mar 2016 02:27:46 -0800 Subject: [PATCH 6/6] Adding upgrade-charm hook --- hooks/pg_gw_hooks.py | 5 +++++ hooks/pg_gw_utils.py | 9 +-------- hooks/upgrade-charm | 1 + 3 files changed, 7 insertions(+), 8 deletions(-) create mode 120000 hooks/upgrade-charm diff --git a/hooks/pg_gw_hooks.py b/hooks/pg_gw_hooks.py index a64030b..5b37e89 100755 --- a/hooks/pg_gw_hooks.py +++ b/hooks/pg_gw_hooks.py @@ -100,6 +100,11 @@ def config_changed(): restart_pg() +@hooks.hook('upgrade-charm') +def upgrade_charm(): + load_iptables() + + @hooks.hook('stop') def stop(): ''' diff --git a/hooks/pg_gw_utils.py b/hooks/pg_gw_utils.py index 47577de..00c3402 100644 --- a/hooks/pg_gw_utils.py +++ b/hooks/pg_gw_utils.py @@ -338,14 +338,7 @@ def load_iptables(): _exec_cmd(['sudo', 'iptables', '-A', 'INPUT', '-p', 'udp', '-j', 'ACCEPT', '-s', network, '-d', network, '-m', 'state', '--state', 'NEW']) - _exec_cmd(['sudo', 'iptables', '-I', 'INPUT', '-s', network, - '-d', '224.0.0.18/32', '-j', 'ACCEPT']) - _exec_cmd(['sudo', 'iptables', '-I', 'INPUT', '-p', 'vrrp', '-j', - 'ACCEPT']) - _exec_cmd(['sudo', 'iptables', '-A', 'INPUT', '-p', 'tcp', '-j', - 'ACCEPT', '-d', config('plumgrid-virtual-ip'), '-m', - 'state', '--state', 'NEW']) - apt_install('iptables-persistent') + apt_install('iptables-persistent') def get_cidr_from_iface(interface): diff --git a/hooks/upgrade-charm b/hooks/upgrade-charm new file mode 120000 index 0000000..3aec9ba --- /dev/null +++ b/hooks/upgrade-charm @@ -0,0 +1 @@ +pg_gw_hooks.py \ No newline at end of file