Merge "Add remaining tempest testcase for Kingbird"

tempest/README.rst

@@ -20,6 +20,9 @@ I) Add kingbird configurations to config file::
         cfg.StrOpt('endpoint_type',
                    default='publicURL',
                    help="Endpoint type of Kingbird service."),
+        cfg.IntOpt('TIME_TO_SYNC',
+                   default=30),
+                   help="Maximum time to wait for a sync call to complete."),
         cfg.StrOpt('endpoint_url',
                    help="Endpoint URL of Kingbird service."),
         cfg.StrOpt('api_version',
@@ -39,12 +42,15 @@ III) Add kingbird_group and KingbirdGroup to list of opts(_opts)
 
    It generates etc/tempest.conf.sample. Copy it to /etc/tempest/ and rename as tempest.conf
 
-4. Copy tempest testcases for Kingbird::
+4. Make sure the default values represented by DEFAULT_QUOTAS in tempest/api/kingbird/base.py
+   has to be same as kingbird_global_limit section in kingbird.conf.
+
+5. Copy tempest testcases for Kingbird::
 
     $ cp -r tempest/tests/api/kingbird <tempest root directory>/tempest/api/
     $ cp tempest/tests/common/kingbird.py <tempest root directory>/tempest/common/
 
-5. Set kingbird = True under [service_available] section in tempest.conf::
+6. Set kingbird = True under [service_available] section in tempest.conf::
 
 To list all Kingbird tempest cases, go to tempest directory, then run::
 

tempest/tests/api/kingbird/base.py

@@ -12,14 +12,29 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-
+import collections
+import time
 
 from tempest.common import kingbird
 from tempest import config
 from tempest.lib.common import api_version_utils
+from tempest.lib.common.utils import data_utils
 import tempest.test
 
 CONF = config.CONF
+Global_instance_limit = 10
+DEFAULT_QUOTAS = {
+    u'quota_set': {
+        u'metadata_items': 128, u'subnet': 10, u'consistencygroups': 10,
+        u'floatingip': 50, u'gigabytes': 1000, u'backup_gigabytes': 1000,
+        u'ram': 51200, u'floating_ips': 10, u'snapshots': 10,
+        u'instances': 10, u'key_pairs': 100, u'volumes': 10, u'router': 10,
+        u'security_group': 10, u'cores': 20, u'backups': 10, u'fixed_ips': -1,
+        u'port': 50, u'security_groups': 10, u'network': 10
+        }
+    }
+# Time to wait for sync to finish
+TIME_TO_SYNC = CONF.kingbird.TIME_TO_SYNC
 
 
 class BaseKingbirdTest(api_version_utils.BaseMicroversionTest,
@@ -35,9 +50,11 @@ class BaseKingbirdTest(api_version_utils.BaseMicroversionTest,
 
     @classmethod
     def setup_credentials(cls):
-        cls.set_network_resources()
         super(BaseKingbirdTest, cls).setup_credentials()
-        cls.auth_token = kingbird.get_keystone_authtoken()
+        session = kingbird.get_session()
+        cls.auth_token = session.get_token()
+        cls.key_client = kingbird.get_key_client(session)
+        cls.regions = kingbird.get_regions(cls.key_client)
 
     @classmethod
     def setup_clients(cls):
@@ -46,10 +63,30 @@ class BaseKingbirdTest(api_version_utils.BaseMicroversionTest,
     @classmethod
     def resource_setup(cls):
         super(BaseKingbirdTest, cls).resource_setup()
+        # Create Project, User, flavor, subnet & network for test
+        project_name = data_utils.rand_name(__name__ + '-project')
+        user_name = data_utils.rand_name(__name__ + '-user')
+        password = data_utils.rand_name(__name__ + '-password')
+        openstack_details = kingbird.get_openstack_drivers(cls.key_client,
+                                                           cls.regions[0],
+                                                           project_name,
+                                                           user_name,
+                                                           password)
+        cls.openstack_drivers = openstack_details['os_drivers']
+        cls.resource_ids = kingbird.create_resources(cls.openstack_drivers)
+        cls.resource_ids.update(openstack_details)
+        cls.session = openstack_details['session']
 
     @classmethod
     def resource_cleanup(cls):
         super(BaseKingbirdTest, cls).resource_cleanup()
+        default_quota = {'instances': DEFAULT_QUOTAS['quota_set']['instances'],
+                         'cores': DEFAULT_QUOTAS['quota_set']['cores'],
+                         'ram': DEFAULT_QUOTAS['quota_set']['ram']}
+        cls.set_default_quota(CONF.kingbird.project_id, default_quota)
+        kingbird.resource_cleanup(cls.openstack_drivers, cls.resource_ids)
+        kingbird.delete_custom_kingbird_quota(
+            cls.auth_token, CONF.kingbird.project_id, None)
 
     def setUp(self):
         super(BaseKingbirdTest, self).setUp()
@@ -95,3 +132,67 @@ class BaseKingbirdTest(api_version_utils.BaseMicroversionTest,
         new_values = kingbird.create_custom_kingbird_quota_wrong_token(
             cls.auth_token, project_id, new_quota_values)
         return new_values
+
+    @classmethod
+    def create_instance(cls, count=1):
+        try:
+            server_ids = kingbird.create_instance(cls.openstack_drivers,
+                                                  cls.resource_ids, count)
+        except Exception as e:
+            server_ids = {'server_ids': list(e.args)}
+            raise
+        finally:
+            cls.resource_ids.update(server_ids)
+
+    @classmethod
+    def delete_instance(cls):
+        kingbird.delete_instance(cls.openstack_drivers, cls.resource_ids)
+        cls.resource_ids['instances'] = None
+
+    @classmethod
+    def calculate_quota_limits(cls, project_id):
+        calculated_quota_limits = collections.defaultdict(dict)
+        resource_usage = kingbird.get_usage_from_os_client(
+            cls.session, cls.regions, project_id)
+        total_usages = cls.get_summation(resource_usage)
+        for current_region in cls.regions:
+            global_remaining_limit = Global_instance_limit - \
+                total_usages['instances']
+            new_limit_for_region = global_remaining_limit + resource_usage[
+                current_region]['instances']
+            calculated_quota_limits.update(
+                {current_region: new_limit_for_region})
+        return calculated_quota_limits
+
+    @classmethod
+    def get_summation(cls, regions_dict):
+        # Adds resources usages from different regions
+        single_region = {}
+        resultant_dict = collections.Counter()
+        for current_region in regions_dict:
+            single_region[current_region] = collections.Counter(
+                regions_dict[current_region])
+            resultant_dict += single_region[current_region]
+        return dict(resultant_dict)
+
+    @classmethod
+    def get_usage_manually(cls, project_id):
+        resource_usage = kingbird.get_usage_from_os_client(
+            cls.session, cls.regions, project_id)
+        resource_usage = cls.get_summation(resource_usage)
+        return {'quota_set': resource_usage}
+
+    @classmethod
+    def get_actual_limits(cls, project_id):
+        actual_limits = kingbird.get_actual_limits(
+            cls.session, cls.regions, project_id)
+        return actual_limits
+
+    @classmethod
+    def wait_sometime_for_sync(cls):
+        time.sleep(TIME_TO_SYNC)
+
+    @classmethod
+    def set_default_quota(cls, project_id, quota_to_set):
+        kingbird.set_default_quota(
+            cls.session, cls.regions, project_id, **quota_to_set)

tempest/tests/api/kingbird/test_kingbird_api.py

@@ -13,103 +13,130 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import uuid
-
 from tempest.api.kingbird import base
 from tempest import config
 
+import novaclient
+
 CONF = config.CONF
-FAKE_PROJECT = str(uuid.uuid4())
-DEFAULT_QUOTAS = {
-    u'quota_set': {
-        u'metadata_items': 128, u'subnet': 10, u'consistencygroups': 10,
-        u'floatingip': 50, u'gigabytes': 1000, u'backup_gigabytes': 1000,
-        u'ram': 51200, u'floating_ips': 10, u'snapshots': 10,
-        u'instances': 10, u'key_pairs': 100, u'volumes': 10, u'router': 10,
-        u'security_group': 10, u'cores': 20, u'backups': 10, u'fixed_ips': -1,
-        u'port': 50, u'security_groups': 10, u'network': 10
-        }
-    }
+DEFAULT_QUOTAS = base.DEFAULT_QUOTAS
 
 
 class KingbirdTestJSON(base.BaseKingbirdTest):
 
     @classmethod
-    def setup_clients(cls):
-        super(KingbirdTestJSON, cls).setup_clients()
+    def setup_clients(self):
+        super(KingbirdTestJSON, self).setup_clients()
 
     def tearDown(self):
         super(KingbirdTestJSON, self).tearDown()
 
+    @classmethod
+    def resource_setup(self):
+        super(KingbirdTestJSON, self).resource_setup()
+        self.PROJECT_ID = self.resource_ids["project_id"]
+
     def test_kingbird_put_method(self):
         new_quota = {"quota_set": {"instances": 15, "cores": 10}}
-        actual_value = self.create_custom_kingbird_quota(FAKE_PROJECT,
+        actual_value = self.create_custom_kingbird_quota(self.PROJECT_ID,
                                                          new_quota)
-        expected_value = {FAKE_PROJECT: new_quota["quota_set"]}
+        expected_value = {self.PROJECT_ID: new_quota["quota_set"]}
         self.assertEqual(expected_value, eval(actual_value))
 
     def test_kingbird_get_method(self):
         new_quota = {"quota_set": {"instances": 15, "cores": 10}}
-        self.create_custom_kingbird_quota(FAKE_PROJECT,
+        self.create_custom_kingbird_quota(self.PROJECT_ID,
                                           new_quota)
-        actual_value = self.get_custom_kingbird_quota(FAKE_PROJECT)
-        new_quota["quota_set"].update({'project_id': FAKE_PROJECT})
+        actual_value = self.get_custom_kingbird_quota(self.PROJECT_ID)
+        new_quota["quota_set"].update({'project_id': self.PROJECT_ID})
         self.assertEqual(new_quota, eval(actual_value))
 
     def test_kingbird_delete_method(self):
         new_quota = {"quota_set": {"instances": 15, "cores": 10}}
         quota_to_delete = {"quota_set": ["cores"]}
-        self.create_custom_kingbird_quota(FAKE_PROJECT,
+        self.create_custom_kingbird_quota(self.PROJECT_ID,
                                           new_quota)
-        self.delete_custom_kingbird_quota(FAKE_PROJECT,
+        self.delete_custom_kingbird_quota(self.PROJECT_ID,
                                           quota_to_delete)
         quota_after_delete = eval(self.get_custom_kingbird_quota(
-            FAKE_PROJECT))
+            self.PROJECT_ID))
         self.assertNotIn("cores", quota_after_delete["quota_set"])
 
     def test_kingbird_delete_all_method(self):
         new_quota = {"quota_set": {"instances": 15, "cores": 10}}
-        self.create_custom_kingbird_quota(FAKE_PROJECT,
+        self.create_custom_kingbird_quota(self.PROJECT_ID,
                                           new_quota)
-        self.delete_custom_kingbird_quota(FAKE_PROJECT)
+        self.delete_custom_kingbird_quota(self.PROJECT_ID)
         actual_quota_after_delete = eval(self.get_custom_kingbird_quota(
-            FAKE_PROJECT))
+            self.PROJECT_ID))
         expected_quota_after_delete = {"quota_set":
-                                       {"project_id": FAKE_PROJECT}}
+                                       {"project_id": self.PROJECT_ID}}
         self.assertEqual(expected_quota_after_delete,
                          actual_quota_after_delete)
 
     def test_kingbird_get_default_method_after_update(self):
         new_quota = {"quota_set": {"instances": 15, "cores": 10}}
-        self.create_custom_kingbird_quota(FAKE_PROJECT,
+        self.create_custom_kingbird_quota(self.PROJECT_ID,
                                           new_quota)
         actual_value = self.get_default_kingbird_quota()
         self.assertEqual(eval(actual_value), DEFAULT_QUOTAS)
-        self.delete_custom_kingbird_quota(FAKE_PROJECT)
-
-    def test_quota_sync_for_project(self):
-        actual_value = self.quota_sync_for_project(FAKE_PROJECT)
-        expected_value = u'triggered quota sync for ' + FAKE_PROJECT
-        self.assertEqual(eval(actual_value), expected_value)
+        self.delete_custom_kingbird_quota(self.PROJECT_ID)
 
     def test_get_quota_usage_for_project(self):
-        actual_usage = self.get_quota_usage_for_project(FAKE_PROJECT)
-        expected_usage = {u'quota_set': {u'key_pairs': 1}}
-        # Assert nova usage, which will be common for all projects
-        self.assertEqual(eval(actual_usage), expected_usage)
+        self.create_instance(count=1)
+        actual_usage = self.get_quota_usage_for_project(self.PROJECT_ID)
+        expected_usage = self.get_usage_manually(self.PROJECT_ID)
+        self.assertEqual(eval(actual_usage)["quota_set"]["ram"],
+                         expected_usage["quota_set"]["ram"])
+        self.assertEqual(eval(actual_usage)["quota_set"]["cores"],
+                         expected_usage["quota_set"]["cores"])
+        self.assertEqual(eval(actual_usage)["quota_set"]["instances"],
+                         expected_usage["quota_set"]["instances"])
+        self.delete_instance()
 
     def test_kingbird_put_method_wrong_token(self):
         new_quota = {"quota_set": {"instances": 15, "cores": 10}}
-        response = self.create_custom_kingbird_quota_wrong_token(FAKE_PROJECT,
-                                                                 new_quota)
+        response = self.create_custom_kingbird_quota_wrong_token(
+            self.PROJECT_ID, new_quota)
         self.assertEqual(response.status_code, 401)
         self.assertEqual(response.text, u'Authentication required')
 
     def test_kingbird_get_default_method_after_delete(self):
         new_quota = {"quota_set": {"instances": 15, "cores": 10}}
-        self.create_custom_kingbird_quota(FAKE_PROJECT,
+        self.create_custom_kingbird_quota(self.PROJECT_ID,
                                           new_quota)
-        self.delete_custom_kingbird_quota(FAKE_PROJECT)
+        self.delete_custom_kingbird_quota(self.PROJECT_ID)
         actual_value = self.get_default_kingbird_quota()
         self.assertEqual(eval(actual_value), DEFAULT_QUOTAS)
-        self.delete_custom_kingbird_quota(FAKE_PROJECT)
+        self.delete_custom_kingbird_quota(self.PROJECT_ID)
+
+    def test_quota_sync_for_project(self):
+        # Delete custom quota if there are any for this project
+        self.delete_custom_kingbird_quota(self.PROJECT_ID)
+        self.create_instance()
+        sync_status = self.quota_sync_for_project(self.PROJECT_ID)
+        expected_status = u"triggered quota sync for " + self.PROJECT_ID
+        calculated_limits = self.calculate_quota_limits(self.PROJECT_ID)
+        self.wait_sometime_for_sync()
+        actual_limits = self.get_actual_limits(self.PROJECT_ID)
+        self.assertEqual(calculated_limits, actual_limits)
+        self.assertEqual(eval(sync_status), expected_status)
+        self.delete_instance()
+
+    def test_quota_exceed_after_sync(self):
+        new_quota = {"quota_set": {"instances": 2}}
+        self.create_custom_kingbird_quota(self.PROJECT_ID,
+                                          new_quota)
+        self.quota_sync_for_project(self.PROJECT_ID)
+        self.wait_sometime_for_sync()
+        try:
+            self.create_instance(count=3)
+        except Exception as exp:
+            self.assertIsInstance(exp, novaclient.exceptions.Forbidden)
+            message = exp.message
+        self.assertIn(u"Quota exceeded for instances", message)
+        self.delete_instance()
+        default_quota = {'instances': DEFAULT_QUOTAS['quota_set']['instances'],
+                         'cores': DEFAULT_QUOTAS['quota_set']['cores'],
+                         'ram': DEFAULT_QUOTAS['quota_set']['ram']}
+        self.set_default_quota(self.PROJECT_ID, default_quota)

tempest/tests/common/kingbird.py

@@ -13,52 +13,91 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+import collections
 import json
 import requests
+import time
 
+from keystoneclient.auth.identity import v3
+from keystoneclient import session
+from keystoneclient.v3 import client as ks_client
+from neutronclient.neutron import client as nt_client
+from novaclient import client as nv_client
 from oslo_log import log as logging
 from tempest import config
 
 CONF = config.CONF
+NOVA_API_VERSION = "2.1"
+NEUTRON_API_VERSION = "2.0"
+FLAVOR_NAME = "kb_test_flavor"
+NETWORK_NAME = "kb_test_network"
+SUBNET_NAME = "kb_test_subnet"
+SERVER_NAME = "kb_test_server"
+SUBNET_RANGE = "192.168.199.0/24"
 
 LOG = logging.getLogger(__name__)
 
 
-def get_keystone_authtoken():
-    headers = {
-        'Content-type': 'application/json',
-    }
-    data = {
-        "auth": {
-            "identity": {
-                "methods": [
-                    "password"
-                    ],
-                "password": {
-                    "user": {
-                        "domain": {
-                            "name": CONF.auth.admin_domain_name
-                            },
-                        "name": CONF.auth.admin_username,
-                        "password": CONF.auth.admin_password
-                        }
-                    }
-                },
-            "scope": {
-                "project": {
-                    "domain": {
-                        "name": CONF.auth.admin_domain_name
-                        },
-                    "name": CONF.auth.admin_tenant_name
-                    }
-                }
-            }
-    }
-    url_string = CONF.identity.uri_v3 + "/auth/tokens"
-    body = json.dumps(data)
-    response = requests.post(url_string, headers=headers, data=body)
-    token = response.headers['X-Subject-Token']
-    return token
+def get_session():
+    return get_current_session(
+        CONF.identity.username,
+        CONF.identity.password,
+        CONF.identity.tenant_name
+        )
+
+
+def get_current_session(username, password, tenant_name):
+    auth = v3.Password(
+        auth_url=CONF.identity.uri_v3,
+        username=username,
+        password=password,
+        project_name=tenant_name,
+        user_domain_name=CONF.identity.domain_name,
+        project_domain_name=CONF.identity.default_domain_id)
+    sess = session.Session(auth=auth)
+    return sess
+
+
+def get_openstack_drivers(key_client, region, project_name, user_name,
+                          password):
+    # Create Project, User and asign role to new user
+    project = key_client.projects.create(project_name,
+                                         CONF.identity.domain_name)
+    user = key_client.users.create(user_name, CONF.identity.domain_name,
+                                   project.id, password)
+    admin_role = [current_role.id for current_role in
+                  key_client.roles.list() if current_role.name == 'admin'][0]
+
+    key_client.roles.grant(admin_role, user=user, project=project)
+    session = get_current_session(user_name, password, project_name)
+    nova_client = nv_client.Client(NOVA_API_VERSION,
+                                   session=session,
+                                   region_name=region)
+    neutron_client = nt_client.Client(NEUTRON_API_VERSION, session=session,
+                                      region_name=region)
+    return {"user_id": user.id, "project_id": project.id, "session": session,
+            "os_drivers": [key_client, nova_client, neutron_client]}
+
+
+def get_key_client(session):
+    return ks_client.Client(session=session)
+
+
+def create_instance(openstack_drivers, resource_ids, count=1):
+    nova_client = openstack_drivers[1]
+    server_ids = []
+    image = nova_client.images.find(id=CONF.compute.image_ref)
+    flavor = nova_client.flavors.find(id=resource_ids['flavor_id'])
+    try:
+        for x in range(count):
+            server = nova_client.servers.create(
+                SERVER_NAME, image, flavor,
+                nics=[{'net-id': resource_ids['network_id']}])
+            server_ids.append(server.id)
+        return {'server_ids': server_ids}
+    except Exception as e:
+        e.args = tuple(server_ids)
+        raise e
 
 
 def get_urlstring_and_headers(token):
@@ -128,3 +167,100 @@ def create_custom_kingbird_quota_wrong_token(token,
     body = json.dumps(new_quota_values)
     response = requests.put(url_string, headers=headers, data=body)
     return response
+
+
+def get_regions(key_client):
+    return [current_region.id for current_region in
+            key_client.regions.list()]
+
+
+def delete_instance(openstack_drivers, resource_ids):
+    nova_client = openstack_drivers[1]
+    if 'server_ids' in resource_ids:
+        for server_id in resource_ids['server_ids']:
+            nova_client.servers.delete(server_id)
+        retries = 6
+        # Delete may take time, So wait(with timeout) till the
+        # instance is deleted
+        while retries > 0:
+            LOG.debug("waiting for instance to get deleted")
+            time.sleep(1)
+            nova_list = [current_server.id for current_server in
+                         nova_client.servers.list()]
+            if len(set(resource_ids['server_ids']) & set(nova_list)):
+                continue
+            else:
+                return
+        LOG.exception('Resource deleting failed, manually delete with IDs %s'
+                      % resource_ids)
+
+
+def resource_cleanup(openstack_drivers, resource_ids):
+    key_client = openstack_drivers[0]
+    nova_client = openstack_drivers[1]
+    neutron_client = openstack_drivers[2]
+    nova_client.flavors.delete(resource_ids['flavor_id'])
+    neutron_client.delete_subnet(resource_ids['subnet_id'])
+    neutron_client.delete_network(resource_ids['network_id'])
+    key_client.projects.delete(resource_ids['project_id'])
+    key_client.users.delete(resource_ids['user_id'])
+
+
+def get_usage_from_os_client(session, regions, project_id):
+    resource_usage_all = collections.defaultdict(dict)
+    for current_region in regions:
+        resource_usage = collections.defaultdict(dict)
+        nova_client = nv_client.Client(NOVA_API_VERSION,
+                                       session=session,
+                                       region_name=current_region)
+        limits = nova_client.limits.get().to_dict()
+        resource_usage['ram'] = limits['absolute']['totalRAMUsed']
+        resource_usage['cores'] = limits['absolute']['totalCoresUsed']
+        resource_usage['instances'] = limits['absolute']['totalInstancesUsed']
+        resource_usage['key_pairs'] = len(nova_client.keypairs.list())
+        resource_usage_all[current_region] = resource_usage
+    return resource_usage_all
+
+
+def get_actual_limits(session, regions, project_id):
+    resource_usage = collections.defaultdict(dict)
+    for current_region in regions:
+        nova_client = nv_client.Client(NOVA_API_VERSION,
+                                       session=session,
+                                       region_name=current_region)
+        updated_quota = nova_client.quotas.get(project_id)
+        resource_usage.update({current_region: updated_quota.instances})
+    return resource_usage
+
+
+def create_resources(openstack_drivers):
+    nova_client = openstack_drivers[1]
+    neutron_client = openstack_drivers[2]
+    flavor = nova_client.flavors.create(
+        FLAVOR_NAME, 128, 1, 1, flavorid='auto')
+    network_body = {'network': {'name': NETWORK_NAME, 'admin_state_up': True}}
+    network = neutron_client.create_network(body=network_body)
+    body_create_subnet = {
+        "subnets": [
+            {
+                'cidr': SUBNET_RANGE,
+                'ip_version': 4,
+                'network_id': network['network']['id'],
+                'name': SUBNET_NAME
+            }
+        ]
+    }
+    subnet = neutron_client.create_subnet(body=body_create_subnet)
+    return {
+        'subnet_id': subnet['subnets'][0]['id'],
+        'network_id': network['network']['id'],
+        'flavor_id': flavor.id
+    }
+
+
+def set_default_quota(session, regions, project_id, **quota_to_set):
+    for current_region in regions:
+        nova_client = nv_client.Client(NOVA_API_VERSION,
+                                       session=session,
+                                       region_name=current_region)
+        nova_client.quotas.update(project_id, **quota_to_set)