diff --git a/config.yaml b/config.yaml index 10cb6149..bff19a67 100644 --- a/config.yaml +++ b/config.yaml @@ -308,3 +308,29 @@ options: for Neutron agents (DHCP and L3 agents). This option overrides the default-availability-zone charm config setting only when the Juju provider sets JUJU_AVAILABILITY_ZONE. + firewall-group-log-output-base: + type: string + default: + description: | + This option allows setting a path for Firewall Group logs. + A valid file system path must be provided. If this option is not + provided Neutron will use syslog as a destination. + (Available from Stein) + firewall-group-log-rate-limit: + type: int + default: + description: | + Log entries are queued for writing to a log file when a packet rate + exceeds the limit set by this option. + Possible values: null (no rate limitation), integer values greater than 100. + WARNING: Should be NOT LESS than 100, if set (if null logging will not be + rate limited). + (Available from Stein) + firewall-group-log-burst-limit: + type: int + default: 25 + description: | + This option sets the maximum queue size for log entries. + Can be used to avoid excessive memory consumption. + WARNING: Should be NOT LESS than 25. + (Available from Stein) diff --git a/hooks/charmhelpers/contrib/network/ovs/__init__.py b/hooks/charmhelpers/contrib/network/ovs/__init__.py index a8856e9c..34755674 100644 --- a/hooks/charmhelpers/contrib/network/ovs/__init__.py +++ b/hooks/charmhelpers/contrib/network/ovs/__init__.py @@ -217,19 +217,35 @@ def full_restart(): service('force-reload-kmod', 'openvswitch-switch') -def enable_ipfix(bridge, target): - '''Enable IPfix on bridge to target. +def enable_ipfix(bridge, target, + cache_active_timeout=60, + cache_max_flows=128, + sampling=64): + '''Enable IPFIX on bridge to target. :param bridge: Bridge to monitor - :param target: IPfix remote endpoint + :param target: IPFIX remote endpoint + :param cache_active_timeout: The maximum period in seconds for + which an IPFIX flow record is cached + and aggregated before being sent + :param cache_max_flows: The maximum number of IPFIX flow records + that can be cached at a time + :param sampling: The rate at which packets should be sampled and + sent to each target collector ''' - cmd = ['ovs-vsctl', 'set', 'Bridge', bridge, 'ipfix=@i', '--', - '--id=@i', 'create', 'IPFIX', 'targets="{}"'.format(target)] + cmd = [ + 'ovs-vsctl', 'set', 'Bridge', bridge, 'ipfix=@i', '--', + '--id=@i', 'create', 'IPFIX', + 'targets="{}"'.format(target), + 'sampling={}'.format(sampling), + 'cache_active_timeout={}'.format(cache_active_timeout), + 'cache_max_flows={}'.format(cache_max_flows), + ] log('Enabling IPfix on {}.'.format(bridge)) subprocess.check_call(cmd) def disable_ipfix(bridge): - '''Diable IPfix on target bridge. + '''Diable IPFIX on target bridge. :param bridge: Bridge to modify ''' cmd = ['ovs-vsctl', 'clear', 'Bridge', bridge, 'ipfix'] diff --git a/hooks/charmhelpers/contrib/openstack/audits/openstack_security_guide.py b/hooks/charmhelpers/contrib/openstack/audits/openstack_security_guide.py index e5b7ac1e..b7b8a60f 100644 --- a/hooks/charmhelpers/contrib/openstack/audits/openstack_security_guide.py +++ b/hooks/charmhelpers/contrib/openstack/audits/openstack_security_guide.py @@ -126,7 +126,11 @@ def _config_ini(path): :returns: Configuration contained in path :rtype: Dict """ - conf = configparser.ConfigParser() + # When strict is enabled, duplicate options are not allowed in the + # parsed INI; however, Oslo allows duplicate values. This change + # causes us to ignore the duplicate values which is acceptable as + # long as we don't validate any multi-value options + conf = configparser.ConfigParser(strict=False) conf.read(path) return dict(conf) @@ -204,7 +208,7 @@ def validate_file_ownership(config): "Invalid ownership configuration: {}".format(key)) owner = options.get('owner', config.get('owner', 'root')) group = options.get('group', config.get('group', 'root')) - optional = options.get('optional', config.get('optional', 'False')) + optional = options.get('optional', config.get('optional', False)) if '*' in file_name: for file in glob.glob(file_name): if file not in files.keys(): @@ -226,7 +230,7 @@ def validate_file_permissions(config): raise RuntimeError( "Invalid ownership configuration: {}".format(key)) mode = options.get('mode', config.get('permissions', '600')) - optional = options.get('optional', config.get('optional', 'False')) + optional = options.get('optional', config.get('optional', False)) if '*' in file_name: for file in glob.glob(file_name): if file not in files.keys(): diff --git a/hooks/charmhelpers/contrib/openstack/cert_utils.py b/hooks/charmhelpers/contrib/openstack/cert_utils.py index 0ba57024..b494af64 100644 --- a/hooks/charmhelpers/contrib/openstack/cert_utils.py +++ b/hooks/charmhelpers/contrib/openstack/cert_utils.py @@ -106,9 +106,11 @@ class CertRequest(object): sans = sorted(list(set(entry['addresses']))) request[entry['cn']] = {'sans': sans} if self.json_encode: - return {'cert_requests': json.dumps(request, sort_keys=True)} + req = {'cert_requests': json.dumps(request, sort_keys=True)} else: - return {'cert_requests': request} + req = {'cert_requests': request} + req['unit_name'] = local_unit().replace('/', '_') + return req def get_certificate_request(json_encode=True): diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 6ccaf14a..5df59efc 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1710,6 +1710,10 @@ class NeutronAPIContext(OSContextGenerator): 'rel_key': 'enable-nsg-logging', 'default': False, }, + 'enable_nfg_logging': { + 'rel_key': 'enable-nfg-logging', + 'default': False, + }, 'global_physnet_mtu': { 'rel_key': 'global-physnet-mtu', 'default': 1500, diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index c57aaf35..a3561760 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -110,17 +110,19 @@ def is_device_mounted(device): return bool(re.search(r'MOUNTPOINT=".+"', out)) -def mkfs_xfs(device, force=False): +def mkfs_xfs(device, force=False, inode_size=1024): """Format device with XFS filesystem. By default this should fail if the device already has a filesystem on it. :param device: Full path to device to format :ptype device: tr :param force: Force operation - :ptype: force: boolean""" + :ptype: force: boolean + :param inode_size: XFS inode size in bytes + :ptype inode_size: int""" cmd = ['mkfs.xfs'] if force: cmd.append("-f") - cmd += ['-i', 'size=1024', device] + cmd += ['-i', "size={}".format(inode_size), device] check_call(cmd) diff --git a/hooks/neutron_contexts.py b/hooks/neutron_contexts.py index 31617d27..61a30cb3 100644 --- a/hooks/neutron_contexts.py +++ b/hooks/neutron_contexts.py @@ -8,6 +8,7 @@ from charmhelpers.core.hookenv import ( related_units, unit_get, network_get_primary_address, + log, ) from charmhelpers.contrib.openstack.context import ( OSContextGenerator, @@ -49,6 +50,9 @@ CORE_PLUGIN = { OVS_ODL: NEUTRON_OVS_ODL_PLUGIN, } +NFG_LOG_RATE_LIMIT_MIN = 100 +NFG_LOG_BURST_LIMIT_MIN = 25 + def _get_availability_zone(): from neutron_utils import get_availability_zone as get_az @@ -107,6 +111,33 @@ class L3AgentContext(OSContextGenerator): return ctxt +def validate_nfg_log_path(desired_nfg_log_path): + if not desired_nfg_log_path: + # None means "we need to use syslog" - no need + # to check anything on filesystem + return None + + dst_dir, _ = os.path.split(desired_nfg_log_path) + path_exists = os.path.exists(dst_dir) + if not path_exists: + log( + "Desired NFG log directory {} not exists! " + "falling back to syslog".format(dst_dir), + "WARN" + ) + return None + + if path_exists and os.path.isdir(desired_nfg_log_path): + log( + "Desired NFG log path {} should be file, not directory! " + "falling back to syslog".format(desired_nfg_log_path), + "WARN" + ) + return None + + return desired_nfg_log_path + + class NeutronGatewayContext(NeutronAPIContext): def __call__(self): @@ -131,6 +162,7 @@ class NeutronGatewayContext(NeutronAPIContext): 'enable_metadata_network': config('enable-metadata-network'), 'enable_isolated_metadata': config('enable-isolated-metadata'), 'availability_zone': _get_availability_zone(), + 'enable_nfg_logging': api_settings['enable_nfg_logging'], } ctxt['local_ip'] = get_local_ip() @@ -161,6 +193,26 @@ class NeutronGatewayContext(NeutronAPIContext): ctxt['enable_metadata_network'] = True ctxt['enable_isolated_metadata'] = True + ctxt['nfg_log_output_base'] = validate_nfg_log_path( + config('firewall-group-log-output-base') + ) + ctxt['nfg_log_rate_limit'] = config( + 'firewall-group-log-rate-limit' + ) + if ctxt['nfg_log_rate_limit'] is not None: + ctxt['nfg_log_rate_limit'] = max( + ctxt['nfg_log_rate_limit'], + NFG_LOG_RATE_LIMIT_MIN + ) + ctxt['nfg_log_burst_limit'] = config( + 'firewall-group-log-burst-limit' + ) + if ctxt['nfg_log_burst_limit'] is not None: + ctxt['nfg_log_burst_limit'] = max( + ctxt['nfg_log_burst_limit'], + NFG_LOG_BURST_LIMIT_MIN + ) + return ctxt diff --git a/hooks/neutron_utils.py b/hooks/neutron_utils.py index d0a9dac4..697dd99d 100644 --- a/hooks/neutron_utils.py +++ b/hooks/neutron_utils.py @@ -155,6 +155,7 @@ GATEWAY_PKGS = { "nova-api-metadata", "neutron-metering-agent", "neutron-lbaas-agent", + "libnetfilter-log1", # fwaas_v2_log ], NSX: [ "neutron-dhcp-agent", @@ -197,6 +198,7 @@ PY3_PACKAGES = [ 'python3-neutron', 'python3-neutron-fwaas', 'python3-neutron-lbaas', + 'python3-zmq', # fwaas_v2_log ] EARLY_PACKAGES = { diff --git a/templates/stein/l3_agent.ini b/templates/stein/l3_agent.ini index 78fee431..c4af5cc4 100644 --- a/templates/stein/l3_agent.ini +++ b/templates/stein/l3_agent.ini @@ -30,4 +30,16 @@ ha_vrrp_health_check_interval = 30 {% endif -%} [AGENT] +{% if enable_nfg_logging -%} +extensions = fwaas_v2,fwaas_v2_log +[network_log] +{% if nfg_log_rate_limit -%} +rate_limit = {{ nfg_log_rate_limit }} +{% endif -%} +burst_limit = {{ nfg_log_burst_limit }} +{% if nfg_log_output_base -%} +local_output_log_base = {{ nfg_log_output_base }} +{% endif -%} +{% else %} extensions = fwaas_v2 +{% endif -%} \ No newline at end of file diff --git a/unit_tests/test_neutron_contexts.py b/unit_tests/test_neutron_contexts.py index 39c78153..0b9323c2 100644 --- a/unit_tests/test_neutron_contexts.py +++ b/unit_tests/test_neutron_contexts.py @@ -156,6 +156,7 @@ class TestNeutronGatewayContext(CharmTestCase): self.config.side_effect = self.test_config.get self.maxDiff = None + @patch.object(neutron_contexts, 'validate_nfg_log_path', lambda x: x) @patch('neutron_utils.config') @patch('charmhelpers.contrib.openstack.context.relation_get') @patch('charmhelpers.contrib.openstack.context.related_units') @@ -168,7 +169,8 @@ class TestNeutronGatewayContext(CharmTestCase): 'enable-l3ha': 'True', 'enable-qos': 'True', 'network-device-mtu': 9000, - 'dns-domain': 'openstack.example.'} + 'dns-domain': 'openstack.example.', + 'enable-nfg-logging': 'True'} self.test_config.set('plugin', 'ovs') self.test_config.set('debug', False) self.test_config.set('verbose', True) @@ -179,6 +181,10 @@ class TestNeutronGatewayContext(CharmTestCase): self.test_config.set('vlan-ranges', 'physnet1:1000:2000 physnet2:2001:3000') self.test_config.set('flat-network-providers', 'physnet3 physnet4') + self.test_config.set('firewall-group-log-output-base', + '/var/log/firewall-logs') + self.test_config.set('firewall-group-log-rate-limit', 100) + self.test_config.set('firewall-group-log-burst-limit', 50) def config_side_effect(key): return { @@ -224,8 +230,13 @@ class TestNeutronGatewayContext(CharmTestCase): 'dhcp-match': 'set:ipxe,175' }, 'availability_zone': 'nova', + 'enable_nfg_logging': True, + 'nfg_log_burst_limit': 50, + 'nfg_log_output_base': '/var/log/firewall-logs', + 'nfg_log_rate_limit': 100, }) + @patch.object(neutron_contexts, 'validate_nfg_log_path', lambda x: x) @patch('neutron_utils.config') @patch('charmhelpers.contrib.openstack.context.relation_get') @patch('charmhelpers.contrib.openstack.context.related_units') @@ -294,6 +305,10 @@ class TestNeutronGatewayContext(CharmTestCase): 'dhcp-match': 'set:ipxe,175' }, 'availability_zone': 'nova', + 'enable_nfg_logging': False, + 'nfg_log_burst_limit': 25, + 'nfg_log_output_base': None, + 'nfg_log_rate_limit': None, }) @patch('charmhelpers.contrib.openstack.context.relation_get') @@ -401,6 +416,19 @@ class TestNeutronGatewayContext(CharmTestCase): self.assertEqual( 'az1', context()['availability_zone']) + @patch('charmhelpers.contrib.openstack.context.relation_get') + @patch('charmhelpers.contrib.openstack.context.related_units') + @patch('charmhelpers.contrib.openstack.context.relation_ids') + @patch.object(neutron_contexts, 'get_shared_secret') + def test_nfg_min_settings(self, _secret, _rids, _runits, _rget): + self.test_config.set('firewall-group-log-rate-limit', 90) + self.test_config.set('firewall-group-log-burst-limit', 20) + self.network_get_primary_address.return_value = '192.168.20.2' + self.unit_get.return_value = '10.5.0.1' + ctxt = neutron_contexts.NeutronGatewayContext()() + self.assertEqual(ctxt['nfg_log_burst_limit'], 25) + self.assertEqual(ctxt['nfg_log_rate_limit'], 100) + class TestSharedSecret(CharmTestCase):