From 03b3fe6f981ff130e08c15c0a0cf92d7d71f5c10 Mon Sep 17 00:00:00 2001 From: Ken Fujimoto Date: Fri, 9 Feb 2024 08:05:03 +0000 Subject: [PATCH] Fix kubernetes resource existence check in v2 API This patch fixes the problem that instantiating a vnf instance succeeds wrongly when a resource made by another instance with the same name exists. This patch adds "tacker_vnf_instance_id" to metadata.labels when creating kubernetes resources in CNF v2 API Instantiate, and when creating or deleting resources, if the resource already exists, it checks if it was created in own vnf instance by metadata.labels. Closes-Bug: #2053098 Change-Id: I4a22fc50706750f9689d89d24c1db28de8e2dc90 --- .../etsi_containerized_vnf_usage_guide.rst | 7 ++++++ .../infra_drivers/kubernetes/helm.py | 4 ++-- .../infra_drivers/kubernetes/kubernetes.py | 21 ++++++++--------- .../kubernetes/kubernetes_common.py | 2 -- .../kubernetes/kubernetes_resource.py | 23 ++++++++++++++++++- .../kubernetes/kubernetes_utils.py | 19 +++++++++++---- .../mgmt_drivers/container_update_mgmt_v2.py | 6 +++-- .../sol_kubernetes_v2/test_vnflcm_basic.py | 13 +++++++++++ .../kubernetes/test_kubernetes.py | 2 +- 9 files changed, 73 insertions(+), 24 deletions(-) diff --git a/doc/source/user/etsi_containerized_vnf_usage_guide.rst b/doc/source/user/etsi_containerized_vnf_usage_guide.rst index 34da17af5..588a04e21 100644 --- a/doc/source/user/etsi_containerized_vnf_usage_guide.rst +++ b/doc/source/user/etsi_containerized_vnf_usage_guide.rst @@ -312,6 +312,13 @@ The following is a simple example of ``deployment`` resource. ``topology_template.node_templates.VDU1.properties.name`` in the helloworld3_df_simple.yaml file. +.. note:: In version 2 API, the ``tacker_vnf_instance_id`` key and + VNF instance ID value are added to the ``metadata.labels`` to + identify which VNF instance created the resource. + Please note that if you have defined the ``tacker_vnf_instance_id`` + label in advance, the value will be overwritten with + the VNF instance ID. + 3. Create a TOSCA.meta File ~~~~~~~~~~~~~~~~~~~~~~~~~~~ The TOSCA.Meta file contains version information for the TOSCA.Meta file, CSAR, diff --git a/tacker/sol_refactored/infra_drivers/kubernetes/helm.py b/tacker/sol_refactored/infra_drivers/kubernetes/helm.py index d81bfc17e..17f3bf002 100644 --- a/tacker/sol_refactored/infra_drivers/kubernetes/helm.py +++ b/tacker/sol_refactored/infra_drivers/kubernetes/helm.py @@ -305,7 +305,7 @@ class Helm(kubernetes_common.KubernetesCommon): def _create_reses_from_manifest(self, k8s_api_client, namespace, k8s_resources): for k8s_res in k8s_resources: - if k8s_res['kind'] in kubernetes_utils.SUPPORTED_NAMESPACE_KINDS: + if k8s_res['kind'] in kubernetes_utils.SUPPORTED_NAMESPACE_KIND: k8s_res.setdefault('metadata', {}) k8s_res['metadata'].setdefault('namespace', namespace) @@ -331,7 +331,7 @@ class Helm(kubernetes_common.KubernetesCommon): # NOTE: {some string} must not include '-'. return {vdu_ids[res.name[:res.name.rfind("-")]]: res for res in k8s_reses - if (res.kind in kubernetes_common.TARGET_KIND + if (res.kind in kubernetes_utils.TARGET_KIND and res.name[:res.name.rfind("-")] in vdu_ids)} def _init_instantiated_vnf_info(self, inst, flavour_id, vdu_reses, diff --git a/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes.py b/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes.py index e00591760..9864a9fb3 100644 --- a/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes.py +++ b/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes.py @@ -28,8 +28,6 @@ LOG = logging.getLogger(__name__) CONF = config.CONF CHECK_INTERVAL = 10 -SCALABLE_KIND = {"Deployment", "ReplicaSet", "StatefulSet"} - class Kubernetes(kubernetes_common.KubernetesCommon): @@ -48,7 +46,7 @@ class Kubernetes(kubernetes_common.KubernetesCommon): k8s_reses, namespace = self._setup_k8s_reses( vnfd, target_k8s_files, k8s_api_client, - req.additionalParams.get('namespace')) + req.additionalParams.get('namespace'), inst.id) vdus_num = self._get_vdus_num_from_grant_req_res_defs( grant_req.addResources) @@ -60,7 +58,7 @@ class Kubernetes(kubernetes_common.KubernetesCommon): f' manifest does not match the VNFD.') continue - if vdu_res.kind in SCALABLE_KIND: + if vdu_res.kind in kubernetes_utils.SCALABLE_KIND: vdu_res.body['spec']['replicas'] = vdus_num[vdu_name] # deploy k8s resources @@ -77,7 +75,7 @@ class Kubernetes(kubernetes_common.KubernetesCommon): self._update_vnfc_info(inst, k8s_api_client) def _setup_k8s_reses(self, vnfd, target_k8s_files, k8s_api_client, - namespace): + namespace, inst_id): # NOTE: this check should be done in STARTING phase. vnf_artifact_files = vnfd.get_vnf_artifact_files() diff_files = set(target_k8s_files) - set(vnf_artifact_files) @@ -87,7 +85,7 @@ class Kubernetes(kubernetes_common.KubernetesCommon): # get k8s content from yaml file return kubernetes_utils.get_k8s_reses_from_json_files( - target_k8s_files, vnfd, k8s_api_client, namespace) + target_k8s_files, vnfd, k8s_api_client, namespace, inst_id) def instantiate_rollback(self, req, inst, grant_req, grant, vnfd): vim_info = inst_utils.select_vim_info(inst.vimConnectionInfo) @@ -103,7 +101,7 @@ class Kubernetes(kubernetes_common.KubernetesCommon): try: k8s_reses, _ = self._setup_k8s_reses( vnfd, target_k8s_files, k8s_api_client, - req.additionalParams.get('namespace')) + req.additionalParams.get('namespace'), inst.id) except sol_ex.SolException: # it means it failed in a basic check and it failes always. # nothing to do since instantiate failed in it too. @@ -113,7 +111,6 @@ class Kubernetes(kubernetes_common.KubernetesCommon): # delete k8s resources body = client.V1DeleteOptions(propagation_policy='Foreground') self._delete_k8s_resource(k8s_reses, body) - # wait k8s resource delete complete self._wait_k8s_reses_deleted(k8s_reses) @@ -136,7 +133,7 @@ class Kubernetes(kubernetes_common.KubernetesCommon): # get k8s content from yaml file namespace = inst.instantiatedVnfInfo.metadata['namespace'] k8s_reses, _ = kubernetes_utils.get_k8s_reses_from_json_files( - target_k8s_files, vnfd, k8s_api_client, namespace) + target_k8s_files, vnfd, k8s_api_client, namespace, inst.id) k8s_reses.reverse() # delete k8s resources @@ -198,7 +195,7 @@ class Kubernetes(kubernetes_common.KubernetesCommon): if vnfc.vduId in target_vdus} k8s_reses, _ = self._setup_k8s_reses( - vnfd, target_k8s_files, k8s_api_client, namespace) + vnfd, target_k8s_files, k8s_api_client, namespace, inst.id) vdu_reses = self._select_vdu_reses( vnfd, inst.instantiatedVnfInfo.flavourId, k8s_reses) @@ -248,7 +245,7 @@ class Kubernetes(kubernetes_common.KubernetesCommon): vdu_reses = [] for vdu_name, vdu_num in vdus_num.items(): vdu_res = self._get_vdu_res(inst, k8s_api_client, vdu_name) - if vdu_res.kind not in SCALABLE_KIND: + if vdu_res.kind not in kubernetes_utils.SCALABLE_KIND: LOG.error(f'scale vdu {vdu_name}' f' is not scalable resource') continue @@ -301,7 +298,7 @@ class Kubernetes(kubernetes_common.KubernetesCommon): # res.name is properties.name itself return {vdu_ids[res.name]: res for res in k8s_reses - if (res.kind in kubernetes_common.TARGET_KIND + if (res.kind in kubernetes_utils.TARGET_KIND and res.name in vdu_ids)} def _init_instantiated_vnf_info(self, inst, flavour_id, vdu_reses, diff --git a/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_common.py b/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_common.py index 7b979107b..ca22e93a0 100644 --- a/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_common.py +++ b/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_common.py @@ -34,8 +34,6 @@ LOG = logging.getLogger(__name__) CONF = config.CONF CHECK_INTERVAL = 10 -TARGET_KIND = {"Pod", "Deployment", "DaemonSet", "StatefulSet", "ReplicaSet"} - class KubernetesCommon(object): diff --git a/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_resource.py b/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_resource.py index b8be97a4e..00c77c8e4 100644 --- a/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_resource.py +++ b/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_resource.py @@ -29,6 +29,7 @@ LOG = logging.getLogger(__name__) CONF = config.CONF CHECK_INTERVAL = 10 +VNF_INSTANCE_ID_LABEL = 'tacker_vnf_instance_id' def convert(name): @@ -47,6 +48,8 @@ class CommonResource: self.name = k8s_res.get('metadata', {}).get('name') self.metadata = k8s_res.get('metadata', {}) self.body = k8s_res + self.inst_id = k8s_res.get('metadata', {}).get('labels', {}).get( + VNF_INSTANCE_ID_LABEL) def create(self): pass @@ -59,7 +62,25 @@ class CommonResource: def is_exists(self): try: - return self.read() is not None + info = self.read() + if info is None: + # resource not exists + return False + # resource exists + # check if the operation uses helm. + # if helm is used, self.inst_id is None. + if self.inst_id is None: + # it means check is not necessary + return True + + # check whether other made it by "metadata.labels" + if info.metadata.labels is None: + # labels not exists. it means made by other + return False + if info.metadata.labels.get(VNF_INSTANCE_ID_LABEL) != self.inst_id: + # made by other + return False + return True except sol_ex.K8sResourceNotFound: return False diff --git a/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_utils.py b/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_utils.py index 45e2f0b96..6b1fb9d36 100644 --- a/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_utils.py +++ b/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_utils.py @@ -31,7 +31,7 @@ from tacker.sol_refactored.infra_drivers.kubernetes import kubernetes_resource LOG = logging.getLogger(__name__) -SUPPORTED_NAMESPACE_KINDS = { +SUPPORTED_NAMESPACE_KIND = { "Binding", "ConfigMap", "ControllerRevision", @@ -55,10 +55,15 @@ SUPPORTED_NAMESPACE_KINDS = { "ServiceAccount", "StatefulSet", } +SCALABLE_KIND = {"Deployment", "ReplicaSet", "StatefulSet"} +TARGET_KIND = {"Pod", "Deployment", "DaemonSet", "StatefulSet", "ReplicaSet"} +UNLABELED_KIND = {"SubjectAccessReview", "LocalSubjectAccessReview", + "SelfSubjectAccessReview", "SelfSubjectRulesReview", + "TokenReview"} def get_k8s_reses_from_json_files(target_k8s_files, vnfd, k8s_api_client, - namespace): + namespace, inst_id): k8s_resources = [] @@ -78,18 +83,24 @@ def get_k8s_reses_from_json_files(target_k8s_files, vnfd, k8s_api_client, for k8s_res in k8s_resources: if not k8s_res.get('kind'): raise sol_ex.K8sInvalidManifestFound() - if k8s_res['kind'] in SUPPORTED_NAMESPACE_KINDS: + if k8s_res['kind'] in SUPPORTED_NAMESPACE_KIND: k8s_res.setdefault('metadata', {}) if namespace is None: k8s_res['metadata'].setdefault('namespace', 'default') else: k8s_res['metadata']['namespace'] = namespace + # Set label to identify the VnfInstance + if k8s_res['kind'] not in UNLABELED_KIND: + k8s_res.setdefault('metadata', {}) + k8s_res['metadata'].setdefault('labels', {}) + k8s_res['metadata']['labels'][ + kubernetes_resource.VNF_INSTANCE_ID_LABEL] = inst_id # check namespace if namespace is None: namespaces = {k8s_res['metadata']['namespace'] for k8s_res in k8s_resources - if k8s_res['kind'] in SUPPORTED_NAMESPACE_KINDS} + if k8s_res['kind'] in SUPPORTED_NAMESPACE_KIND} if len(namespaces) > 1: raise sol_ex.NamespaceNotUniform() namespace = namespaces.pop() if namespaces else 'default' diff --git a/tacker/sol_refactored/mgmt_drivers/container_update_mgmt_v2.py b/tacker/sol_refactored/mgmt_drivers/container_update_mgmt_v2.py index 0ff7b3068..ed4230be9 100644 --- a/tacker/sol_refactored/mgmt_drivers/container_update_mgmt_v2.py +++ b/tacker/sol_refactored/mgmt_drivers/container_update_mgmt_v2.py @@ -226,14 +226,16 @@ class ContainerUpdateMgmtDriver(kubernetes.Kubernetes): old_vnfd = vnfd_utils.Vnfd(uuidutils.generate_uuid()) old_vnfd.init_from_csar_dir(self.old_csar_dir) old_k8s_objs, _ = self._setup_k8s_reses( - old_vnfd, target_k8s_files, k8s_api_client, namespace) + old_vnfd, target_k8s_files, k8s_api_client, namespace, + vnf_instance['id']) # Get new_k8s_objs target_k8s_files = new_inst_vnf_info['metadata'][ 'lcm-kubernetes-def-files'] new_vnfd = vnfd_utils.Vnfd(self.req['vnfdId']) new_vnfd.init_from_csar_dir(self.new_csar_dir) new_k8s_objs, _ = self._setup_k8s_reses( - new_vnfd, target_k8s_files, k8s_api_client, namespace) + new_vnfd, target_k8s_files, k8s_api_client, namespace, + vnf_instance['id']) # Initialize k8s_pod_objs and k8s_config_objs k8s_pod_objs = [] k8s_config_objs = [] diff --git a/tacker/tests/functional/sol_kubernetes_v2/test_vnflcm_basic.py b/tacker/tests/functional/sol_kubernetes_v2/test_vnflcm_basic.py index f286737bd..0ebc68223 100644 --- a/tacker/tests/functional/sol_kubernetes_v2/test_vnflcm_basic.py +++ b/tacker/tests/functional/sol_kubernetes_v2/test_vnflcm_basic.py @@ -42,6 +42,11 @@ class VnfLcmKubernetesTest(base_v2.BaseVnfLcmKubernetesV2Test): def setUp(self): super(VnfLcmKubernetesTest, self).setUp() + def _get_vdu_label(self, inst_vnf_info, vdu_id): + vdu_reses = inst_vnf_info['metadata']['vdu_reses'] + return vdu_reses[vdu_id]['metadata'].get( + 'labels', {}).get('tacker_vnf_instance_id') + def test_basic_lcms_max(self): """Test LCM operations with all attributes set @@ -147,6 +152,14 @@ class VnfLcmKubernetesTest(base_v2.BaseVnfLcmKubernetesV2Test): expected = {'VDU1': 1, 'VDU2': 2, 'VDU3': 1, 'VDU5': 1, 'VDU6': 1} self.assertEqual(expected, vdu_nums) + # check VDU label + inst_vnf_info = body['instantiatedVnfInfo'] + self.assertEqual(inst_id, self._get_vdu_label(inst_vnf_info, 'VDU1')) + self.assertEqual(inst_id, self._get_vdu_label(inst_vnf_info, 'VDU2')) + self.assertEqual(inst_id, self._get_vdu_label(inst_vnf_info, 'VDU3')) + self.assertEqual(inst_id, self._get_vdu_label(inst_vnf_info, 'VDU5')) + self.assertEqual(inst_id, self._get_vdu_label(inst_vnf_info, 'VDU6')) + # 4. Scale out a VNF instance scale_out_req = paramgen.max_sample_scale_out() resp, body = self.scale_vnf_instance(inst_id, scale_out_req) diff --git a/tacker/tests/unit/sol_refactored/infra_drivers/kubernetes/test_kubernetes.py b/tacker/tests/unit/sol_refactored/infra_drivers/kubernetes/test_kubernetes.py index cd3f1a083..a46b4d864 100644 --- a/tacker/tests/unit/sol_refactored/infra_drivers/kubernetes/test_kubernetes.py +++ b/tacker/tests/unit/sol_refactored/infra_drivers/kubernetes/test_kubernetes.py @@ -52,7 +52,7 @@ class TestKubernetes(base.TestCase): target_k8s_files = [not_exist] ex = self.assertRaises(sol_ex.CnfDefinitionNotFound, self.driver._setup_k8s_reses, self.vnfd_1, - target_k8s_files, mock.Mock(), mock.Mock()) + target_k8s_files, mock.Mock(), mock.Mock(), mock.Mock()) self.assertEqual(expected_ex.detail, ex.detail) def test_wait_k8s_reses_ready(self):