From 88e38e8e9493757d5db4da2b69dcc033b2f7465b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Dulko?= Date: Wed, 13 Feb 2019 10:04:08 +0100 Subject: [PATCH] Add option to tag Neutron resources created by us If we consider a K8s cluster running on OpenStack VM's, which is a perfect use case for Kuryr-Kubernetes, we can easily imagine creating multiple clusters in a single OpenStack public or private cloud. In such use case those K8s clusters may come and go. As Kuryr is creating some OpenStack resources, such as ports, networks, subnets, floating IP's or SG's, it's useful to have a way of identifying those Kuryr-created resources to delete them along with the K8s cluster that used them. This commit makes that possible by adding an option to add tag to all Neutron resources created by Kuryr. Change-Id: If75028e17d13ec62fb414fa9797ee7ac02d948d1 Implements: blueprint kuryr-resources-tagging --- kuryr_kubernetes/config.py | 7 +++++++ kuryr_kubernetes/controller/drivers/lbaasv2.py | 3 +++ .../controller/drivers/namespace_security_groups.py | 2 ++ .../controller/drivers/namespace_subnet.py | 3 +++ .../controller/drivers/nested_macvlan_vif.py | 2 ++ .../controller/drivers/nested_vlan_vif.py | 5 ++++- .../controller/drivers/network_policy.py | 1 + kuryr_kubernetes/controller/drivers/neutron_vif.py | 4 +++- kuryr_kubernetes/controller/drivers/public_ip.py | 13 ++++++++----- kuryr_kubernetes/controller/drivers/sriov.py | 1 + kuryr_kubernetes/controller/drivers/utils.py | 13 +++++++++++++ .../unit/controller/drivers/test_neutron_vif.py | 4 ++-- 12 files changed, 49 insertions(+), 9 deletions(-) diff --git a/kuryr_kubernetes/config.py b/kuryr_kubernetes/config.py index 1e0a976f6..00d0c931e 100644 --- a/kuryr_kubernetes/config.py +++ b/kuryr_kubernetes/config.py @@ -205,6 +205,13 @@ neutron_defaults = [ help=_("A mapping of default subnets for certain driverType " "in a form of :"), default={}), + cfg.ListOpt('resource_tags', + help=_("List of tags that will be applied to all OpenStack " + "(Neutron and Octavia) resources created by Kuryr. " + "This can be used to identify and garbage-collect " + "them when Kubernetes cluster Kuryr was serving is no " + "longer needed."), + default=[]) ] octavia_defaults = [ diff --git a/kuryr_kubernetes/controller/drivers/lbaasv2.py b/kuryr_kubernetes/controller/drivers/lbaasv2.py index 9692fcfdc..e4b264c9e 100644 --- a/kuryr_kubernetes/controller/drivers/lbaasv2.py +++ b/kuryr_kubernetes/controller/drivers/lbaasv2.py @@ -29,6 +29,7 @@ from oslo_utils import timeutils from kuryr_kubernetes import clients from kuryr_kubernetes import config from kuryr_kubernetes.controller.drivers import base +from kuryr_kubernetes.controller.drivers import utils as c_utils from kuryr_kubernetes import exceptions as k_exc from kuryr_kubernetes.objects import lbaas as obj_lbaas from kuryr_kubernetes import utils @@ -119,6 +120,7 @@ class LBaaSv2Driver(base.LBaaSDriver): }, }) sg_id = sg['security_group']['id'] + c_utils.tag_neutron_resources('security-groups', [sg_id]) loadbalancer.security_groups.append(sg_id) vip_port = self._get_vip_port(loadbalancer) neutron.update_port( @@ -290,6 +292,7 @@ class LBaaSv2Driver(base.LBaaSDriver): }, }) sg_id = sg['security_group']['id'] + c_utils.tag_neutron_resources('security-groups', [sg_id]) loadbalancer.security_groups.append(sg_id) vip_port = self._get_vip_port(loadbalancer) neutron.update_port( diff --git a/kuryr_kubernetes/controller/drivers/namespace_security_groups.py b/kuryr_kubernetes/controller/drivers/namespace_security_groups.py index 1485f36e4..477f5a3d1 100644 --- a/kuryr_kubernetes/controller/drivers/namespace_security_groups.py +++ b/kuryr_kubernetes/controller/drivers/namespace_security_groups.py @@ -21,6 +21,7 @@ from kuryr_kubernetes import clients from kuryr_kubernetes import config from kuryr_kubernetes import constants from kuryr_kubernetes.controller.drivers import base +from kuryr_kubernetes.controller.drivers import utils from kuryr_kubernetes import exceptions from neutronclient.common import exceptions as n_exc @@ -107,6 +108,7 @@ class NamespacePodSecurityGroupsDriver(base.PodSecurityGroupsDriver): "project_id": project_id } }).get('security_group') + utils.tag_neutron_resources('security-groups', [sg['id']]) neutron.create_security_group_rule( { "security_group_rule": { diff --git a/kuryr_kubernetes/controller/drivers/namespace_subnet.py b/kuryr_kubernetes/controller/drivers/namespace_subnet.py index 554c3b980..abc84db03 100644 --- a/kuryr_kubernetes/controller/drivers/namespace_subnet.py +++ b/kuryr_kubernetes/controller/drivers/namespace_subnet.py @@ -19,6 +19,7 @@ from oslo_log import log as logging from kuryr_kubernetes import clients from kuryr_kubernetes import constants from kuryr_kubernetes.controller.drivers import default_subnet +from kuryr_kubernetes.controller.drivers import utils as c_utils from kuryr_kubernetes import exceptions from kuryr_kubernetes import utils @@ -121,6 +122,7 @@ class NamespacePodSubnetDriver(default_subnet.DefaultPodSubnetDriver): "project_id": project_id } }).get('network') + c_utils.tag_neutron_resources('networks', [neutron_net['id']]) # create a subnet within that network neutron_subnet = neutron.create_subnet( @@ -134,6 +136,7 @@ class NamespacePodSubnetDriver(default_subnet.DefaultPodSubnetDriver): "project_id": project_id } }).get('subnet') + c_utils.tag_neutron_resources('subnets', [neutron_subnet['id']]) # connect the subnet to the router neutron.add_interface_router(router_id, diff --git a/kuryr_kubernetes/controller/drivers/nested_macvlan_vif.py b/kuryr_kubernetes/controller/drivers/nested_macvlan_vif.py index 8af25cef1..4fea813bc 100755 --- a/kuryr_kubernetes/controller/drivers/nested_macvlan_vif.py +++ b/kuryr_kubernetes/controller/drivers/nested_macvlan_vif.py @@ -19,6 +19,7 @@ from oslo_log import log as logging from kuryr_kubernetes import clients from kuryr_kubernetes.controller.drivers import nested_vif +from kuryr_kubernetes.controller.drivers import utils from kuryr_kubernetes import exceptions as k_exc from kuryr_kubernetes import os_vif_util as ovu @@ -37,6 +38,7 @@ class NestedMacvlanPodVIFDriver(nested_vif.NestedPodVIFDriver): security_groups) vm_port = self._get_parent_port(neutron, pod) container_port = neutron.create_port(req).get('port') + utils.tag_neutron_resources('ports', [container_port['id']]) container_mac = container_port['mac_address'] container_ips = frozenset(entry['ip_address'] for entry in diff --git a/kuryr_kubernetes/controller/drivers/nested_vlan_vif.py b/kuryr_kubernetes/controller/drivers/nested_vlan_vif.py index 3554152eb..afee77df8 100644 --- a/kuryr_kubernetes/controller/drivers/nested_vlan_vif.py +++ b/kuryr_kubernetes/controller/drivers/nested_vlan_vif.py @@ -44,6 +44,7 @@ class NestedVlanPodVIFDriver(nested_vif.NestedPodVIFDriver): rq = self._get_port_request(pod, project_id, subnets, security_groups) port = neutron.create_port(rq).get('port') + utils.tag_neutron_resources('ports', [port['id']]) vlan_id = self._add_subport(neutron, trunk_id, port['id']) return ovu.neutron_to_osvif_vif_nested_vlan(port, subnets, vlan_id) @@ -78,12 +79,14 @@ class NestedVlanPodVIFDriver(nested_vif.NestedPodVIFDriver): LOG.error("There are no vlan ids available to create subports") return [] - bulk_port_rq = {'ports': [port_rq for _ in range(len(subports_info))]} + bulk_port_rq = {'ports': [port_rq] * len(subports_info)} try: ports = neutron.create_port(bulk_port_rq).get('ports') except n_exc.NeutronClientException: LOG.exception("Error creating bulk ports: %s", bulk_port_rq) raise + utils.tag_neutron_resources('ports', [port['id'] for port in ports]) + for index, port in enumerate(ports): subports_info[index]['port_id'] = port['id'] diff --git a/kuryr_kubernetes/controller/drivers/network_policy.py b/kuryr_kubernetes/controller/drivers/network_policy.py index 07964d898..92b73aa81 100644 --- a/kuryr_kubernetes/controller/drivers/network_policy.py +++ b/kuryr_kubernetes/controller/drivers/network_policy.py @@ -167,6 +167,7 @@ class NetworkPolicyDriver(base.NetworkPolicyDriver): # Create initial security group sg = self.neutron.create_security_group(body=security_group_body) sg_id = sg['security_group']['id'] + driver_utils.tag_neutron_resources('security-groups', [sg_id]) i_rules, e_rules = self.parse_network_policy_rules(policy, sg_id) for i_rule in i_rules: sgr_id = driver_utils.create_security_group_rule(i_rule) diff --git a/kuryr_kubernetes/controller/drivers/neutron_vif.py b/kuryr_kubernetes/controller/drivers/neutron_vif.py index 1c314de64..16a1e48b3 100644 --- a/kuryr_kubernetes/controller/drivers/neutron_vif.py +++ b/kuryr_kubernetes/controller/drivers/neutron_vif.py @@ -37,6 +37,7 @@ class NeutronPodVIFDriver(base.PodVIFDriver): rq = self._get_port_request(pod, project_id, subnets, security_groups) port = neutron.create_port(rq).get('port') + utils.tag_neutron_resources('ports', [port['id']]) vif_plugin = self._get_vif_plugin(port) return ovu.neutron_to_osvif_vif(vif_plugin, port, subnets) @@ -48,12 +49,13 @@ class NeutronPodVIFDriver(base.PodVIFDriver): rq = self._get_port_request(pod, project_id, subnets, security_groups, unbound=True) - bulk_port_rq = {'ports': [rq for _ in range(num_ports)]} + bulk_port_rq = {'ports': [rq] * num_ports} try: ports = neutron.create_port(bulk_port_rq).get('ports') except n_exc.NeutronClientException: LOG.exception("Error creating bulk ports: %s", bulk_port_rq) raise + utils.tag_neutron_resources('ports', [port['id'] for port in ports]) vif_plugin = self._get_vif_plugin(ports[0]) diff --git a/kuryr_kubernetes/controller/drivers/public_ip.py b/kuryr_kubernetes/controller/drivers/public_ip.py index 42fac7620..1c1520f5b 100644 --- a/kuryr_kubernetes/controller/drivers/public_ip.py +++ b/kuryr_kubernetes/controller/drivers/public_ip.py @@ -13,10 +13,13 @@ # License for the specific language governing permissions and limitations # under the License. import abc -from kuryr_kubernetes import clients +import six + from neutronclient.common import exceptions as n_exc from oslo_log import log as logging -import six + +from kuryr_kubernetes import clients +from kuryr_kubernetes.controller.drivers import utils LOG = logging.getLogger(__name__) @@ -132,13 +135,13 @@ class FipPubIpDriver(BasePubIpDriver): request['floatingip']['description'] = description try: - response = neutron.create_floatingip(request) + fip = neutron.create_floatingip(request).get('floatingip') except n_exc.NeutronClientException: LOG.exception("Failed to create floating IP - netid=%s ", pub_net_id) raise - return response['floatingip']['id'], response[ - 'floatingip']['floating_ip_address'] + utils.tag_neutron_resources('networks', [fip['id']]) + return fip['id'], fip['floating_ip_address'] def free_ip(self, res_id): neutron = clients.get_neutron_client() diff --git a/kuryr_kubernetes/controller/drivers/sriov.py b/kuryr_kubernetes/controller/drivers/sriov.py index 2e3f9f5e6..ba8d39640 100644 --- a/kuryr_kubernetes/controller/drivers/sriov.py +++ b/kuryr_kubernetes/controller/drivers/sriov.py @@ -50,6 +50,7 @@ class SriovVIFDriver(neutron_vif.NeutronPodVIFDriver): subnets, security_groups) port = neutron.create_port(rq).get('port') + c_utils.tag_neutron_resources('ports', [port['id']]) vif = ovu.neutron_to_osvif_vif(vif_plugin, port, subnets) vif.physnet = physnet diff --git a/kuryr_kubernetes/controller/drivers/utils.py b/kuryr_kubernetes/controller/drivers/utils.py index dc758cd09..0e3ebcc25 100644 --- a/kuryr_kubernetes/controller/drivers/utils.py +++ b/kuryr_kubernetes/controller/drivers/utils.py @@ -379,3 +379,16 @@ def get_namespace_subnet_cidr(namespace): LOG.exception("Kubernetes Client Exception.") raise return net_crd['spec']['subnetCIDR'] + + +def tag_neutron_resources(resource, res_ids): + tags = CONF.neutron_defaults.resource_tags + if tags: + neutron = clients.get_neutron_client() + for res_id in res_ids: + try: + neutron.replace_tag(resource, res_id, body={"tags": tags}) + except n_exc.NeutronClientException: + LOG.warning("Failed to tag %s %s with %s. Ignoring, but this " + "is still unexpected.", resource, res_id, tags, + exc_info=True) diff --git a/kuryr_kubernetes/tests/unit/controller/drivers/test_neutron_vif.py b/kuryr_kubernetes/tests/unit/controller/drivers/test_neutron_vif.py index 84c7a1249..ca225d2ea 100644 --- a/kuryr_kubernetes/tests/unit/controller/drivers/test_neutron_vif.py +++ b/kuryr_kubernetes/tests/unit/controller/drivers/test_neutron_vif.py @@ -39,7 +39,7 @@ class NeutronPodVIFDriver(test_base.TestCase): project_id = mock.sentinel.project_id subnets = mock.sentinel.subnets security_groups = mock.sentinel.security_groups - port = mock.sentinel.port + port = {'id': '910b1183-1f4a-450a-a298-0e80ad06ec8b'} port_request = mock.sentinel.port_request vif = mock.sentinel.vif vif_plugin = mock.sentinel.vif_plugin @@ -72,7 +72,7 @@ class NeutronPodVIFDriver(test_base.TestCase): port_request = mock.sentinel.port_request m_driver._get_port_request.return_value = port_request - port = mock.sentinel.port + port = {'id': '910b1183-1f4a-450a-a298-0e80ad06ec8b'} vif_plugin = mock.sentinel.vif_plugin vif = mock.sentinel.vif bulk_rq = {'ports': [port_request for _ in range(num_ports)]}