Flag tests requiring multiple tenants and users
Guiding core priciple number 7 from the DefCore core definition is [1]: "7. Tests can be remotely or self-administered" Tests that require multiple tenants and users create a barrier to remote adminstration and independent verification of test results. Requiring users to enroll for multiple tenant-isolated accounts on a public cloud creates a resource cost that may be insurmountable for end users, especially when comparing multiple public clouds. In addition, tenant isolation is less a capability for writing interoperable applications, and more a security concern for cloud implementers. While important, the goal of interopearbility is orthgonal to the security goals of tenant isolation. The flagged tests in this patch require multiple tenant-isloated accounts, and are marked to to be fixed where possible, or removed where not possible. This list was determined by successfully running the defcore suite against a test cloud[2], then re-running the tests configured with only one account against the same test cloud[3]. At the January QA meetup, this test list was refined by refactoring Server Negative tests into two classes, one that requires multi-tenant access, one one that does not[4]. This significantly reduced the number of flagged tests.[5] [1] https://github.com/openstack/defcore/blob/master/doc/source/process/CoreDefinition.rst [2] https://refstack.openstack.org/#/results/0b8ac51d-3288-42d8-b276-4a54046e41bd [3] https://refstack.openstack.org/#/results/01f29358-8870-4e15-8f7a-f6a082d44e29 [4] https://review.openstack.org/#/c/283819/ [5] https://etherpad.openstack.org/p/multi-tenant-defcore-tests Change-Id: I5ec801ae48e8220abdaecebe43e77465684e8b5b
This commit is contained in:
Chris Hoge
parent
376e53fc2a
commit
184ff3e19e
35
2015.07.json
35
2015.07.json
|
|
@ -158,7 +158,12 @@
|
|||
"required-since": "2014.07",
|
||||
"tests": {
|
||||
"tempest.api.compute.test_authorization.AuthorizationTestJSON.test_set_metadata_of_alt_account_server_fails": {
|
||||
"idempotent_id": "id-c5f52351-53d9-4fc9-83e5-917f7f5e3d71"
|
||||
"idempotent_id": "id-c5f52351-53d9-4fc9-83e5-917f7f5e3d71",
|
||||
"flagged": {
|
||||
"reason": "D400 This test may create a barrier for independent testing of public clouds",
|
||||
"action": "This test should be changed to require only one tenant or removed if it requires tenant isolation",
|
||||
"date": "2015-12-03"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -490,7 +495,12 @@
|
|||
"required-since": "2014.07",
|
||||
"tests": {
|
||||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_a_server_of_another_tenant": {
|
||||
"idempotent_id": "id-5c75009d-3eea-423e-bea3-61b09fd25f9c"
|
||||
"idempotent_id": "id-5c75009d-3eea-423e-bea3-61b09fd25f9c",
|
||||
"flagged": {
|
||||
"reason": "D400 This test may create a barrier for independent testing of public clouds",
|
||||
"action": "This test should be changed to require only one tenant or removed if it requires tenant isolation",
|
||||
"date": "2015-12-03"
|
||||
}
|
||||
},
|
||||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_id_exceeding_length_limit": {
|
||||
"idempotent_id": "id-f4d7279b-5fd2-4bf2-9ba4-ae35df0d18c5"
|
||||
|
|
@ -1053,7 +1063,12 @@
|
|||
"idempotent_id": "id-5c8e244c-dada-4590-9944-749c455b431f"
|
||||
},
|
||||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_of_another_tenant": {
|
||||
"idempotent_id": "id-543d84c1-dd2e-4c6d-8cb2-b9da0efaa384"
|
||||
"idempotent_id": "id-543d84c1-dd2e-4c6d-8cb2-b9da0efaa384",
|
||||
"flagged": {
|
||||
"reason": "D400 This test may create a barrier for independent testing of public clouds",
|
||||
"action": "This test should be changed to require only one tenant or removed if it requires tenant isolation",
|
||||
"date": "2015-12-03"
|
||||
}
|
||||
},
|
||||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_set_empty_name": {
|
||||
"idempotent_id": "id-38204696-17c6-44da-9590-40f87fb5a899"
|
||||
|
|
@ -1268,10 +1283,20 @@
|
|||
"required-since": "2014.07",
|
||||
"tests": {
|
||||
"tempest.api.object_storage.test_object_services.PublicObjectTest.test_access_public_container_object_without_using_creds": {
|
||||
"idempotent_id": "id-07c9cf95-c0d4-4b49-b9c8-0ef2c9b27193"
|
||||
"idempotent_id": "id-07c9cf95-c0d4-4b49-b9c8-0ef2c9b27193",
|
||||
"flagged": {
|
||||
"reason": "D400 This test may create a barrier for independent testing of public clouds",
|
||||
"action": "This test should be changed to require only one tenant or removed if it requires tenant isolation",
|
||||
"date": "2015-12-03"
|
||||
}
|
||||
},
|
||||
"tempest.api.object_storage.test_object_services.PublicObjectTest.test_access_public_object_with_another_user_creds": {
|
||||
"idempotent_id": "id-54e2a2fe-42dc-491b-8270-8e4217dd4cdc"
|
||||
"idempotent_id": "id-54e2a2fe-42dc-491b-8270-8e4217dd4cdc",
|
||||
"flagged": {
|
||||
"reason": "D400 This test may create a barrier for independent testing of public clouds",
|
||||
"action": "This test should be changed to require only one tenant or removed if it requires tenant isolation",
|
||||
"date": "2015-12-03"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
14
2016.01.json
14
2016.01.json
|
|
@ -635,7 +635,12 @@
|
|||
"required-since": "2014.07",
|
||||
"tests": {
|
||||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_a_server_of_another_tenant": {
|
||||
"idempotent_id": "id-5c75009d-3eea-423e-bea3-61b09fd25f9c"
|
||||
"idempotent_id": "id-5c75009d-3eea-423e-bea3-61b09fd25f9c",
|
||||
"flagged": {
|
||||
"reason": "D400 This test may create a barrier for independent testing of public clouds",
|
||||
"action": "This test should be changed to require only one tenant or removed if it requires tenant isolation",
|
||||
"date": "2015-12-03"
|
||||
}
|
||||
},
|
||||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_id_exceeding_length_limit": {
|
||||
"idempotent_id": "id-f4d7279b-5fd2-4bf2-9ba4-ae35df0d18c5"
|
||||
|
|
@ -1190,7 +1195,12 @@
|
|||
"idempotent_id": "id-5c8e244c-dada-4590-9944-749c455b431f"
|
||||
},
|
||||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_of_another_tenant": {
|
||||
"idempotent_id": "id-543d84c1-dd2e-4c6d-8cb2-b9da0efaa384"
|
||||
"idempotent_id": "id-543d84c1-dd2e-4c6d-8cb2-b9da0efaa384",
|
||||
"flagged": {
|
||||
"reason": "D400 This test may create a barrier for independent testing of public clouds",
|
||||
"action": "This test should be changed to require only one tenant or removed if it requires tenant isolation",
|
||||
"date": "2015-12-03"
|
||||
}
|
||||
},
|
||||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_set_empty_name": {
|
||||
"idempotent_id": "id-38204696-17c6-44da-9590-40f87fb5a899"
|
||||
|
|
|
|||
14
next.json
14
next.json
|
|
@ -231,14 +231,6 @@
|
|||
"project": "nova",
|
||||
"required-since": "2014.07",
|
||||
"tests": {
|
||||
"tempest.api.compute.test_authorization.AuthorizationTestJSON.test_set_metadata_of_alt_account_server_fails": {
|
||||
"idempotent_id": "id-c5f52351-53d9-4fc9-83e5-917f7f5e3d71",
|
||||
"flagged": {
|
||||
"date": "2015-05-06",
|
||||
"reason": "Test implicitly depends on Glance v1 API, which is a SUPPORTED but not the CURRENT version. Vendors may have Glance v1 disabled in favor of Glance v2.",
|
||||
"action": "Implement v2 support for Glance v2 API in Nova. Fix implicit dependency on Glance v1 in test."
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"compute-images-create": {
|
||||
|
|
@ -636,9 +628,6 @@
|
|||
"project": "nova",
|
||||
"required-since": "2014.07",
|
||||
"tests": {
|
||||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_a_server_of_another_tenant": {
|
||||
"idempotent_id": "id-5c75009d-3eea-423e-bea3-61b09fd25f9c"
|
||||
},
|
||||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_id_exceeding_length_limit": {
|
||||
"idempotent_id": "id-f4d7279b-5fd2-4bf2-9ba4-ae35df0d18c5"
|
||||
},
|
||||
|
|
@ -1185,9 +1174,6 @@
|
|||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_name_length_exceeds_256": {
|
||||
"idempotent_id": "id-5c8e244c-dada-4590-9944-749c455b431f"
|
||||
},
|
||||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_of_another_tenant": {
|
||||
"idempotent_id": "id-543d84c1-dd2e-4c6d-8cb2-b9da0efaa384"
|
||||
},
|
||||
"tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_set_empty_name": {
|
||||
"idempotent_id": "id-38204696-17c6-44da-9590-40f87fb5a899"
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue