openstack
/
trove
Code Issues Proposed changes

Allow for invalid packet sequence in keepalive 

In the SQLAlchemy keep_alive class, MariaDB is failing
as pymysql reports an invalid packet sequence.
MariaDB seems to timeout the client in a different
way than MySQL and PXC, which manifests itself as the
aforementioned invalid sequence.  It is now handled
as a special-case exception.

With this fix, the MariaDB scenario tests now pass.

The scenario tests were also tweaked a bit, which aided
in the testing of the fix.  'group=instance' was created,
plus instance_error properly interleaved with
instance_create.  _has_status now calls get_instance with
the admin client so that any faults are accompanied by
a relevant stack trace.  Cases where the result code
was being checked out-of-sequence were removed, and explicit
calls to check the http code for the right client were added.

The replication error messages for promote and eject were
enhanced as well to attempt to debug spurious failures.
One of those failures was 'Replication is not on after 60 seconds.'
This was fixed by setting 'MASTER_CONNECT_RETRY' in the mariadb
gtid replication strategy as was done in:
https://review.openstack.org/#/c/188933

Finally, backup_incremental was added to MariaDB supported
groups and cleaned up elsewhere.

Closes-Bug: #1621702
Change-Id: Id6bde5a34e1d79eece3084f761dcd153c38ccbad
This commit is contained in:
Peter Stachowski 2016-08-29 19:47:47 +00:00
parent 8ba72a5f3f
commit bd761989ee
18 changed files with 240 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
trove/guestagent/datastore/mysql_common/service.py
View File

@ -25,6 +25,7 @@ import uuid
from oslo_log import log as logging
from oslo_utils import encodeutils
from pymysql import err as pymysql_err
from six.moves import urllib
import sqlalchemy
from sqlalchemy import exc
@ -568,6 +569,14 @@ class BaseKeepAliveConnection(interfaces.PoolListener):
raise exc.DisconnectionError()
else:
raise
# MariaDB seems to timeout the client in a different
# way than MySQL and PXC, which manifests itself as
# an invalid packet sequence. Handle it as well.
except pymysql_err.InternalError as ex:
if "Packet sequence number wrong" in ex.message:
raise exc.DisconnectionError()
else:
raise
@six.add_metaclass(abc.ABCMeta)

1
trove/guestagent/strategies/replication/experimental/mariadb_gtid.py
View File

@ -55,6 +55,7 @@ class MariaDBGTIDReplication(mysql_base.MysqlReplicationBase):
"MASTER_PORT=%(port)s, "
"MASTER_USER='%(user)s', "
"MASTER_PASSWORD='%(password)s', "
"MASTER_CONNECT_RETRY=15, "
"MASTER_USE_GTID=slave_pos" %
{
'host': snapshot['master']['host'],

75
trove/taskmanager/manager.py
View File

@ -120,41 +120,43 @@ class Manager(periodic_task.PeriodicTasks):
# and possibly some number of "orphaned" slaves
exception_replicas = []
error_messages = ""
for replica in replica_models:
try:
if replica.id != master_candidate.id:
replica.detach_replica(old_master, for_failover=True)
replica.attach_replica(master_candidate)
except exception.TroveError:
msg = _("promote-to-replica-source: Unable to migrate "
"replica %(slave)s from old replica source "
"%(old_master)s to new source %(new_master)s.")
msg_values = {
"slave": replica.id,
"old_master": old_master.id,
"new_master": master_candidate.id
}
LOG.exception(msg % msg_values)
except exception.TroveError as ex:
msg = (_("Unable to migrate replica %(slave)s from "
"old replica source %(old_master)s to "
"new source %(new_master)s on promote.") %
{"slave": replica.id,
"old_master": old_master.id,
"new_master": master_candidate.id})
LOG.exception(msg)
exception_replicas.append(replica)
error_messages += "%s (%s)\n" % (msg, ex)
try:
old_master.demote_replication_master()
except Exception:
LOG.exception(_("Exception demoting old replica source"))
except Exception as ex:
msg = (_("Exception demoting old replica source %s.") %
old_master.id)
LOG.exception(msg)
exception_replicas.append(old_master)
error_messages += "%s (%s)\n" % (msg, ex)
self._set_task_status([old_master] + replica_models,
InstanceTasks.NONE)
if exception_replicas:
self._set_task_status(exception_replicas,
InstanceTasks.PROMOTION_ERROR)
msg = _("promote-to-replica-source %(id)s: The following "
"replicas may not have been switched: %(replicas)s")
msg_values = {
"id": master_candidate.id,
"replicas": exception_replicas
}
raise ReplicationSlaveAttachError(msg % msg_values)
msg = (_("promote-to-replica-source %(id)s: The following "
"replicas may not have been switched: %(replicas)s") %
{"id": master_candidate.id,
"replicas": [repl.id for repl in exception_replicas]})
raise ReplicationSlaveAttachError("%s:\n%s" %
(msg, error_messages))
with EndNotification(context):
master_candidate = BuiltInstanceTasks.load(context, instance_id)
@ -207,35 +209,34 @@ class Manager(periodic_task.PeriodicTasks):
old_master.attach_public_ips(slave_ips)
exception_replicas = []
error_messages = ""
for replica in replica_models:
try:
if replica.id != master_candidate.id:
replica.detach_replica(old_master, for_failover=True)
replica.attach_replica(master_candidate)
except exception.TroveError:
msg = _("eject-replica-source: Unable to migrate "
"replica %(slave)s from old replica source "
"%(old_master)s to new source %(new_master)s.")
msg_values = {
"slave": replica.id,
"old_master": old_master.id,
"new_master": master_candidate.id
}
LOG.exception(msg % msg_values)
exception_replicas.append(replica.id)
except exception.TroveError as ex:
msg = (_("Unable to migrate replica %(slave)s from "
"old replica source %(old_master)s to "
"new source %(new_master)s on eject.") %
{"slave": replica.id,
"old_master": old_master.id,
"new_master": master_candidate.id})
LOG.exception(msg)
exception_replicas.append(replica)
error_messages += "%s (%s)\n" % (msg, ex)
self._set_task_status([old_master] + replica_models,
InstanceTasks.NONE)
if exception_replicas:
self._set_task_status(exception_replicas,
InstanceTasks.EJECTION_ERROR)
msg = _("eject-replica-source %(id)s: The following "
"replicas may not have been switched: %(replicas)s")
msg_values = {
"id": master_candidate.id,
"replicas": exception_replicas
}
raise ReplicationSlaveAttachError(msg % msg_values)
msg = (_("eject-replica-source %(id)s: The following "
"replicas may not have been switched: %(replicas)s") %
{"id": master_candidate.id,
"replicas": [repl.id for repl in exception_replicas]})
raise ReplicationSlaveAttachError("%s:\n%s" %
(msg, error_messages))
with EndNotification(context):
master = BuiltInstanceTasks.load(context, instance_id)

145
trove/tests/int_tests.py
View File

@ -177,6 +177,10 @@ guest_log_groups.extend([guest_log_group.GROUP])
instance_actions_groups = list(instance_create_groups)
instance_actions_groups.extend([instance_actions_group.GROUP])
instance_groups = list(instance_actions_groups)
instance_groups.extend([instance_error_create_group.GROUP,
instance_force_delete_group.GROUP])
module_groups = list(instance_create_groups)
module_groups.extend([module_group.GROUP])
@ -197,9 +201,8 @@ user_actions_groups = list(instance_create_groups)
user_actions_groups.extend([user_actions_group.GROUP])
# groups common to all datastores
common_groups = list(instance_actions_groups)
common_groups.extend([guest_log_groups, instance_error_create_groups,
instance_force_delete_groups, module_groups])
common_groups = list(instance_groups)
common_groups.extend([guest_log_groups, module_groups])
# Register: Component based groups
register(["backup"], backup_groups)
@ -210,11 +213,12 @@ register(["configuration"], configuration_groups)
register(["configuration_create"], configuration_create_groups)
register(["database"], database_actions_groups)
register(["guest_log"], guest_log_groups)
register(["instance", "instance_actions"], instance_actions_groups)
register(["instance"], instance_groups)
register(["instance_actions"], instance_actions_groups)
register(["instance_create"], instance_create_groups)
register(["instance_error_create"], instance_error_create_groups)
register(["instance_upgrade"], instance_upgrade_groups)
register(["instance_error"], instance_error_create_groups)
register(["instance_force_delete"], instance_force_delete_groups)
register(["instance_upgrade"], instance_upgrade_groups)
register(["module"], module_groups)
register(["module_create"], module_create_groups)
register(["replication"], replication_groups)
@ -223,35 +227,100 @@ register(["root"], root_actions_groups)
register(["user"], user_actions_groups)
# Register: Datastore based groups
# These should contain all functionality currently supported by the datastore
register(["db2_supported"], common_groups,
database_actions_groups, user_actions_groups, configuration_groups)
register(["cassandra_supported"], common_groups,
user_actions_groups, database_actions_groups,
backup_groups, configuration_groups, cluster_actions_groups)
register(["couchbase_supported"], common_groups, backup_groups,
root_actions_groups)
register(["couchdb_supported"], common_groups, backup_groups,
user_actions_groups, database_actions_groups, root_actions_groups)
register(["postgresql_supported"], common_groups,
backup_groups, database_actions_groups, configuration_groups,
root_actions_groups, user_actions_groups,
backup_incremental_groups, replication_groups)
register(["mysql_supported", "percona_supported"], common_groups,
backup_groups, configuration_groups, database_actions_groups,
replication_promote_groups, instance_upgrade_groups,
root_actions_groups, user_actions_groups, backup_incremental_groups)
register(["mariadb_supported"], common_groups,
backup_groups, cluster_actions_groups, configuration_groups,
database_actions_groups, replication_promote_groups,
root_actions_groups, user_actions_groups)
register(["mongodb_supported"], common_groups,
backup_groups, cluster_actions_groups, configuration_groups,
database_actions_groups, root_actions_groups, user_actions_groups)
register(["pxc_supported"], common_groups,
backup_groups, configuration_groups, database_actions_groups,
cluster_actions_groups, root_actions_groups, user_actions_groups)
register(["redis_supported"], common_groups,
backup_groups, replication_promote_groups, cluster_actions_groups)
register(["vertica_supported"], common_groups,
cluster_actions_groups, root_actions_groups, configuration_groups)
# These should contain all functionality currently supported by the datastore.
# Keeping them in alphabetical order may reduce the number of merge conflicts.
register(
["db2_supported"], common_groups,
configuration_groups,
database_actions_groups,
user_actions_groups,
)
register(
["cassandra_supported"], common_groups,
backup_groups,
database_actions_groups,
cluster_actions_groups,
configuration_groups,
user_actions_groups,
)
register(
["couchbase_supported"], common_groups,
backup_groups,
root_actions_groups,
)
register(
["couchdb_supported"], common_groups,
backup_groups,
database_actions_groups,
root_actions_groups,
user_actions_groups,
)
register(
["postgresql_supported"], common_groups,
backup_incremental_groups,
database_actions_groups,
configuration_groups,
replication_groups,
root_actions_groups,
user_actions_groups,
)
register(
["mysql_supported", "percona_supported"], common_groups,
backup_incremental_groups,
configuration_groups,
database_actions_groups,
instance_upgrade_groups,
replication_promote_groups,
root_actions_groups,
user_actions_groups,
)
register(
["mariadb_supported"], common_groups,
backup_incremental_groups,
cluster_actions_groups,
configuration_groups,
database_actions_groups,
replication_promote_groups,
root_actions_groups,
user_actions_groups,
)
register(
["mongodb_supported"], common_groups,
backup_groups,
cluster_actions_groups,
configuration_groups,
database_actions_groups,
root_actions_groups,
user_actions_groups,
)
register(
["pxc_supported"], common_groups,
backup_incremental_groups,
cluster_actions_groups,
configuration_groups,
database_actions_groups,
root_actions_groups,
user_actions_groups,
)
register(
["redis_supported"], common_groups,
backup_groups,
cluster_actions_groups,
replication_promote_groups,
)
register(
["vertica_supported"], common_groups,
cluster_actions_groups,
configuration_groups,
root_actions_groups,
)

3
trove/tests/scenario/groups/instance_create_group.py
View File

@ -58,7 +58,8 @@ class InstanceCreateGroup(TestGroup):
@test(depends_on_groups=[groups.INST_CREATE],
groups=[GROUP, groups.INST_CREATE_WAIT],
runs_after_groups=[groups.MODULE_CREATE, groups.CFGGRP_CREATE])
runs_after_groups=[groups.MODULE_CREATE, groups.CFGGRP_CREATE,
groups.INST_ERROR_CREATE_WAIT])
class InstanceCreateWaitGroup(TestGroup):
"""Test that Instance Create Completes."""

12
trove/tests/scenario/runners/backup_runners.py
View File

@ -260,8 +260,7 @@ class BackupRunner(TestRunner):
self.unauth_client.backups.get, self.backup_info.id)
# we're using a different client, so we'll check the return code
# on it explicitly, instead of depending on 'assert_raises'
self.assert_client_code(expected_http_code=expected_http_code,
client=self.unauth_client)
self.assert_client_code(expected_http_code, client=self.unauth_client)
def run_add_data_for_inc_backup_1(self):
self.backup_host = self.get_instance_host()
@ -304,7 +303,7 @@ class BackupRunner(TestRunner):
def assert_restore_from_backup(self, backup_ref, suffix='',
expected_http_code=200):
result = self._restore_from_backup(backup_ref, suffix=suffix)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self.assert_equal('BUILD', result.status,
'Unexpected instance status')
return result.id
@ -374,7 +373,7 @@ class BackupRunner(TestRunner):
def assert_delete_restored_instance(
self, instance_id, expected_http_code):
self.auth_client.instances.delete(instance_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
def run_delete_restored_inc_1_instance(self, expected_http_code=202):
self.assert_delete_restored_instance(
@ -412,8 +411,7 @@ class BackupRunner(TestRunner):
self.unauth_client.backups.delete, self.backup_info.id)
# we're using a different client, so we'll check the return code
# on it explicitly, instead of depending on 'assert_raises'
self.assert_client_code(expected_http_code=expected_http_code,
client=self.unauth_client)
self.assert_client_code(expected_http_code, client=self.unauth_client)
def run_delete_inc_2_backup(self, expected_http_code=202):
self.assert_delete_backup(
@ -423,7 +421,7 @@ class BackupRunner(TestRunner):
def assert_delete_backup(
self, backup_id, expected_http_code):
self.auth_client.backups.delete(backup_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self._wait_until_backup_is_gone(backup_id)
def _wait_until_backup_is_gone(self, backup_id):

9
trove/tests/scenario/runners/cluster_actions_runners.py
View File

@ -139,7 +139,7 @@ class ClusterActionsRunner(TestRunner):
def assert_cluster_list(self, expected_count,
expected_http_code):
count = len(self.auth_client.clusters.list())
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self.assert_equal(expected_count, count, "Unexpected cluster count")
def run_cluster_show(self, expected_http_code=200,
@ -328,7 +328,8 @@ class ClusterActionsRunner(TestRunner):
self, cluster_id, expected_task_name, expected_http_code,
check_locality=True):
if expected_http_code is not None:
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code,
client=self.auth_client)
if expected_task_name:
self._assert_cluster_response(cluster_id, expected_task_name,
check_locality=check_locality)
@ -365,7 +366,7 @@ class ClusterActionsRunner(TestRunner):
def _assert_cluster_response(self, cluster_id, expected_task_name,
expected_http_code=200, check_locality=True):
cluster = self.auth_client.clusters.get(cluster_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self._assert_cluster_values(cluster, expected_task_name,
check_locality=check_locality)
@ -401,7 +402,7 @@ class ClusterActionsRunner(TestRunner):
"Cluster '%s' still existed after %s seconds."
% (cluster_id, self._time_since(t0)))
except exceptions.NotFound:
self.assert_client_code(404)
self.assert_client_code(404, client=self.auth_client)
class CassandraClusterActionsRunner(ClusterActionsRunner):

13
trove/tests/scenario/runners/configuration_runners.py
View File

@ -134,7 +134,7 @@ class ConfigurationRunner(TestRunner):
description,
datastore=self.instance_info.dbaas_datastore,
datastore_version=self.instance_info.dbaas_datastore_version)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
with TypeCheck('Configuration', result) as configuration:
configuration.has_field('name', basestring)
@ -473,7 +473,7 @@ class ConfigurationRunner(TestRunner):
def assert_group_delete(self, group_id, expected_http_code):
self.auth_client.configurations.delete(group_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
def run_delete_non_dynamic_group(self, expected_http_code=202):
if self.non_dynamic_group_id:
@ -514,15 +514,14 @@ class ConfigurationRunner(TestRunner):
datastore_version=self.instance_info.dbaas_datastore_version,
availability_zone="nova",
configuration=config_id)
self.assert_client_code(200)
self.assert_client_code(200, client=self.auth_client)
self.assert_equal("BUILD", result.status, 'Unexpected inst status')
return result.id
def run_wait_for_conf_instance(
self, expected_states=['BUILD', 'ACTIVE'], expected_http_code=200):
self, expected_states=['BUILD', 'ACTIVE']):
if self.config_inst_id:
self.assert_instance_action(self.config_inst_id, expected_states,
expected_http_code)
self.assert_instance_action(self.config_inst_id, expected_states)
self.create_test_helper_on_instance(self.config_inst_id)
inst = self.auth_client.instances.get(self.config_inst_id)
self.assert_equal(self.config_id_for_inst,
@ -546,7 +545,7 @@ class ConfigurationRunner(TestRunner):
def assert_delete_conf_instance(self, instance_id, expected_http_code):
self.auth_client.instances.delete(instance_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
def run_wait_for_delete_conf_instance(
self, expected_last_state=['SHUTDOWN']):

11
trove/tests/scenario/runners/database_actions_runners.py
View File

@ -53,7 +53,7 @@ class DatabaseActionsRunner(TestRunner):
def assert_databases_create(self, instance_id, serial_databases_def,
expected_http_code):
self.auth_client.databases.create(instance_id, serial_databases_def)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self.wait_for_database_create(instance_id, serial_databases_def)
return serial_databases_def
@ -64,7 +64,7 @@ class DatabaseActionsRunner(TestRunner):
def assert_databases_list(self, instance_id, expected_database_defs,
expected_http_code, limit=2):
full_list = self.auth_client.databases.list(instance_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
listed_databases = {database.name: database for database in full_list}
self.assert_is_none(full_list.next,
"Unexpected pagination in the list.")
@ -86,7 +86,7 @@ class DatabaseActionsRunner(TestRunner):
# Test list pagination.
list_page = self.auth_client.databases.list(instance_id, limit=limit)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self.assert_true(len(list_page) <= limit)
if len(full_list) > limit:
@ -104,7 +104,8 @@ class DatabaseActionsRunner(TestRunner):
"in the page.")
list_page = self.auth_client.databases.list(
instance_id, marker=marker)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code,
client=self.auth_client)
self.assert_pagination_match(
list_page, full_list, limit, len(full_list))
@ -163,7 +164,7 @@ class DatabaseActionsRunner(TestRunner):
database_name,
expected_http_code):
self.auth_client.databases.delete(instance_id, database_name)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self._wait_for_database_delete(instance_id, database_name)
def _wait_for_database_delete(self, instance_id, deleted_database_name):

25
trove/tests/scenario/runners/guest_log_runners.py
View File

@ -100,7 +100,7 @@ class GuestLogRunner(TestRunner):
self.report.log("Executing log_show for log '%s'" % log_name)
log_details = client.instances.log_show(
self.instance_info.id, log_name)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=client)
self.assert_log_details(
log_details, log_name,
expected_type=expected_type,
@ -183,7 +183,7 @@ class GuestLogRunner(TestRunner):
self.report.log("Executing log_enable for log '%s'" % log_name)
log_details = client.instances.log_enable(
self.instance_info.id, log_name)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=client)
self.assert_log_details(
log_details, log_name,
expected_type=expected_type,
@ -200,7 +200,7 @@ class GuestLogRunner(TestRunner):
(log_name, discard))
log_details = client.instances.log_disable(
self.instance_info.id, log_name, discard=discard)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=client)
self.assert_log_details(
log_details, log_name,
expected_type=expected_type,
@ -218,7 +218,7 @@ class GuestLogRunner(TestRunner):
(log_name, disable, discard))
log_details = client.instances.log_publish(
self.instance_info.id, log_name, disable=disable, discard=discard)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=client)
self.assert_log_details(
log_details, log_name,
expected_type=expected_type,
@ -234,7 +234,7 @@ class GuestLogRunner(TestRunner):
self.report.log("Executing log_discard for log '%s'" % log_name)
log_details = client.instances.log_discard(
self.instance_info.id, log_name)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=client)
self.assert_log_details(
log_details, log_name,
expected_type=expected_type,
@ -302,7 +302,7 @@ class GuestLogRunner(TestRunner):
client.instances.log_enable,
self.instance_info.id, log_name)
# we may not be using the main client, so check explicitly here
self.assert_client_code(expected_http_code, client)
self.assert_client_code(expected_http_code, client=client)
def run_test_log_disable_sys(self,
expected_exception=exceptions.BadRequest,
@ -320,7 +320,7 @@ class GuestLogRunner(TestRunner):
self.instance_info.id, log_name,
discard=discard)
# we may not be using the main client, so check explicitly here
self.assert_client_code(expected_http_code, client)
self.assert_client_code(expected_http_code, client=client)
def run_test_log_show_unauth_user(self,
expected_exception=exceptions.NotFound,
@ -337,7 +337,7 @@ class GuestLogRunner(TestRunner):
client.instances.log_show,
self.instance_info.id, log_name)
# we may not be using the main client, so check explicitly here
self.assert_client_code(expected_http_code, client)
self.assert_client_code(expected_http_code, client=client)
def run_test_log_list_unauth_user(self,
expected_exception=exceptions.NotFound,
@ -346,7 +346,7 @@ class GuestLogRunner(TestRunner):
self.unauth_client.instances.log_list,
self.instance_info.id)
# we're not using the main client, so check explicitly here
self.assert_client_code(expected_http_code, self.unauth_client)
self.assert_client_code(expected_http_code, client=self.unauth_client)
def run_test_log_generator_unauth_user(self):
self.assert_log_generator_unauth_user(
@ -406,7 +406,7 @@ class GuestLogRunner(TestRunner):
self.instance_info.id, log_name,
disable=disable, discard=discard)
# we may not be using the main client, so check explicitly here
self.assert_client_code(expected_http_code, client)
self.assert_client_code(expected_http_code, client=client)
def run_test_log_discard_unexposed_user(
self, expected_exception=exceptions.BadRequest,
@ -423,7 +423,7 @@ class GuestLogRunner(TestRunner):
client.instances.log_discard,
self.instance_info.id, log_name)
# we may not be using the main client, so check explicitly here
self.assert_client_code(expected_http_code, client)
self.assert_client_code(expected_http_code, client=client)
def run_test_log_enable_user(self):
expected_status = guest_log.LogStatus.Ready.name
@ -468,7 +468,8 @@ class GuestLogRunner(TestRunner):
expected_states = ['RESTART_REQUIRED', 'ACTIVE']
self.assert_instance_action(instance_id, expected_states, None)
self.auth_client.instances.restart(instance_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code,
client=self.auth_client)
def run_test_wait_for_restart(self, expected_states=['REBOOT', 'ACTIVE']):
if self.test_helper.log_enable_requires_restart():

2
trove/tests/scenario/runners/instance_actions_runners.py
View File

@ -88,7 +88,7 @@ class InstanceActionsRunner(TestRunner):
(resize_flavor_id, instance_id))
self.auth_client.instances.resize_instance(
instance_id, resize_flavor_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
def run_wait_for_instance_resize_flavor(
self, expected_states=['RESIZE', 'ACTIVE']):

9
trove/tests/scenario/runners/instance_create_runners.py
View File

@ -73,7 +73,8 @@ class InstanceCreateRunner(TestRunner):
"Configuration group used by instance create tests.",
datastore=self.instance_info.dbaas_datastore,
datastore_version=self.instance_info.dbaas_datastore_version)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code,
client=self.auth_client)
self.config_group_id = result.id
else:
@ -309,7 +310,8 @@ class InstanceCreateRunner(TestRunner):
def run_initialized_instance_delete(self, expected_http_code=202):
if self.init_inst_id:
self.auth_client.instances.delete(self.init_inst_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code,
client=self.auth_client)
else:
raise SkipTest("Cleanup is not required.")
@ -331,7 +333,8 @@ class InstanceCreateRunner(TestRunner):
def run_initial_configuration_delete(self, expected_http_code=202):
if self.config_group_id:
self.auth_client.configurations.delete(self.config_group_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code,
client=self.auth_client)
else:
raise SkipTest("Cleanup is not required.")
self.config_group_id = None

2
trove/tests/scenario/runners/instance_delete_runners.py
View File

@ -36,7 +36,7 @@ class InstanceDeleteRunner(TestRunner):
self.report.log("Testing delete on instance: %s" % instance_id)
self.auth_client.instances.delete(instance_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
def run_instance_delete_wait(self, expected_states=['SHUTDOWN']):
if self.has_do_not_delete_instance:

13
trove/tests/scenario/runners/instance_error_create_runners.py
View File

@ -40,7 +40,7 @@ class InstanceErrorCreateRunner(TestRunner):
nics=self.instance_info.nics,
datastore=self.instance_info.dbaas_datastore,
datastore_version=self.instance_info.dbaas_datastore_version)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self.error_inst_id = inst.id
def run_create_error2_instance(self, expected_http_code=200):
@ -57,7 +57,7 @@ class InstanceErrorCreateRunner(TestRunner):
nics=self.instance_info.nics,
datastore=self.instance_info.dbaas_datastore,
datastore_version=self.instance_info.dbaas_datastore_version)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self.error2_inst_id = inst.id
def run_wait_for_error_instances(self, expected_states=['ERROR']):
@ -75,7 +75,8 @@ class InstanceErrorCreateRunner(TestRunner):
if not self.error_inst_id:
raise SkipTest("No error instance created.")
instance = self.get_instance(self.error_inst_id)
instance = self.get_instance(
self.error_inst_id, self.auth_client)
with CheckInstance(instance._info) as check:
check.fault()
@ -101,10 +102,12 @@ class InstanceErrorCreateRunner(TestRunner):
def run_delete_error_instances(self, expected_http_code=202):
if self.error_inst_id:
self.auth_client.instances.delete(self.error_inst_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code,
client=self.auth_client)
if self.error2_inst_id:
self.auth_client.instances.delete(self.error2_inst_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code,
client=self.auth_client)
def run_wait_for_error_delete(self, expected_states=['SHUTDOWN']):
delete_ids = []

19
trove/tests/scenario/runners/module_runners.py
View File

@ -386,8 +386,7 @@ class ModuleRunner(TestRunner):
self.unauth_client.modules.get, self.main_test_module.id)
# we're using a different client, so we'll check the return code
# on it explicitly, instead of depending on 'assert_raises'
self.assert_client_code(expected_http_code=expected_http_code,
client=self.unauth_client)
self.assert_client_code(expected_http_code, client=self.unauth_client)
def run_module_list(self):
self.assert_module_list(
@ -686,7 +685,7 @@ class ModuleRunner(TestRunner):
def assert_module_list_instance(self, client, instance_id, expected_count,
expected_http_code=200):
module_list = client.instances.modules(instance_id)
self.assert_client_code(expected_http_code, client)
self.assert_client_code(expected_http_code, client=client)
count = len(module_list)
self.assert_equal(expected_count, count,
"Wrong number of modules from list instance")
@ -701,7 +700,7 @@ class ModuleRunner(TestRunner):
def assert_module_instances(self, client, module_id, expected_count,
expected_http_code=200):
instance_list = client.modules.instances(module_id)
self.assert_client_code(expected_http_code, client)
self.assert_client_code(expected_http_code, client=client)
count = len(instance_list)
self.assert_equal(expected_count, count,
"Wrong number of instances applied from module")
@ -713,7 +712,7 @@ class ModuleRunner(TestRunner):
def assert_module_query(self, client, instance_id, expected_count,
expected_http_code=200, expected_results=None):
modquery_list = client.instances.module_query(instance_id)
self.assert_client_code(expected_http_code, client)
self.assert_client_code(expected_http_code, client=client)
count = len(modquery_list)
self.assert_equal(expected_count, count,
"Wrong number of modules from query")
@ -736,7 +735,7 @@ class ModuleRunner(TestRunner):
expected_http_code=200):
module_apply_list = client.instances.module_apply(
instance_id, [module.id])
self.assert_client_code(expected_http_code, client)
self.assert_client_code(expected_http_code, client=client)
admin_only = (not module.visible or module.auto_apply or
not module.tenant_id)
expected_status = expected_status or 'OK'
@ -856,7 +855,7 @@ class ModuleRunner(TestRunner):
nics=self.instance_info.nics,
modules=[module_id],
)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
return inst.id
def run_module_delete_applied(
@ -873,7 +872,7 @@ class ModuleRunner(TestRunner):
def assert_module_remove(self, client, instance_id, module_id,
expected_http_code=200):
client.instances.module_remove(instance_id, module_id)
self.assert_client_code(expected_http_code, client)
self.assert_client_code(expected_http_code, client=client)
def run_wait_for_inst_with_mods(self, expected_states=['BUILD', 'ACTIVE']):
self.assert_instance_action(self.mod_inst_id, expected_states, None)
@ -903,7 +902,7 @@ class ModuleRunner(TestRunner):
prefix = 'contents'
modretrieve_list = client.instances.module_retrieve(
instance_id, directory=temp_dir, prefix=prefix)
self.assert_client_code(expected_http_code, client)
self.assert_client_code(expected_http_code, client=client)
count = len(modretrieve_list)
self.assert_equal(expected_count, count,
"Wrong number of modules from retrieve")
@ -964,7 +963,7 @@ class ModuleRunner(TestRunner):
def assert_delete_instance(self, instance_id, expected_http_code):
self.auth_client.instances.delete(instance_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
def run_wait_for_delete_inst_with_mods(
self, expected_last_state=['SHUTDOWN']):

21
trove/tests/scenario/runners/replication_runners.py
View File

@ -69,7 +69,7 @@ class ReplicationRunner(TestRunner):
datastore=self.instance_info.dbaas_datastore,
datastore_version=self.instance_info.dbaas_datastore_version,
locality='anti-affinity').id
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
def run_create_single_replica(self, expected_http_code=200):
self.master_backup_count = len(
@ -87,7 +87,7 @@ class ReplicationRunner(TestRunner):
datastore_version=self.instance_info.dbaas_datastore_version,
nics=self.instance_info.nics,
replica_count=replica_count)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
return replica.id
def run_wait_for_single_replica(self, expected_states=['BUILD', 'ACTIVE']):
@ -98,8 +98,8 @@ class ReplicationRunner(TestRunner):
self.replica_1_host = self.get_instance_host(self.replica_1_id)
def _assert_is_master(self, instance_id, replica_ids):
instance = self.get_instance(instance_id)
self.assert_client_code(200)
instance = self.get_instance(instance_id, client=self.admin_client)
self.assert_client_code(200, client=self.admin_client)
CheckInstance(instance._info).slaves()
self.assert_true(
set(replica_ids).issubset(self._get_replica_set(instance_id)))
@ -110,8 +110,8 @@ class ReplicationRunner(TestRunner):
return set([replica['id'] for replica in instance._info['replicas']])
def _assert_is_replica(self, instance_id, master_id):
instance = self.get_instance(instance_id)
self.assert_client_code(200)
instance = self.get_instance(instance_id, client=self.admin_client)
self.assert_client_code(200, client=self.admin_client)
CheckInstance(instance._info).replica_of()
self.assert_equal(master_id, instance._info['replica_of']['id'],
'Unexpected replication master ID')
@ -145,7 +145,7 @@ class ReplicationRunner(TestRunner):
datastore_version=self.instance_info.dbaas_datastore_version,
replica_of=self.non_affinity_master_id,
replica_count=1).id
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
def run_create_multiple_replicas(self, expected_http_code=200):
self.replica_2_id = self.assert_replica_create(
@ -176,7 +176,8 @@ class ReplicationRunner(TestRunner):
else [instance_ids])
for instance_id in instance_ids:
self.auth_client.instances.delete(instance_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code,
client=self.auth_client)
def run_wait_for_delete_non_affinity_repl(
self, expected_last_status=['SHUTDOWN']):
@ -344,8 +345,8 @@ class ReplicationRunner(TestRunner):
replica_id, expected_states, expected_http_code)
def _assert_is_not_replica(self, instance_id):
instance = self.get_instance(instance_id)
self.assert_client_code(200)
instance = self.get_instance(instance_id, client=self.admin_client)
self.assert_client_code(200, client=self.admin_client)
if 'replica_of' not in instance._info:
try:

2
trove/tests/scenario/runners/test_runners.py
View File

@ -572,7 +572,7 @@ class TestRunner(object):
def _has_status(self, instance_id, status, fast_fail_status=None):
fast_fail_status = fast_fail_status or []
instance = self.get_instance(instance_id)
instance = self.get_instance(instance_id, self.admin_client)
self.report.log("Polling instance '%s' for state '%s', was '%s'."
% (instance_id, status, instance.status))
if instance.status in fast_fail_status:

21
trove/tests/scenario/runners/user_actions_runners.py
View File

@ -66,7 +66,7 @@ class UserActionsRunner(TestRunner):
def assert_users_create(self, instance_id, serial_users_def,
expected_http_code):
self.auth_client.users.create(instance_id, serial_users_def)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self.wait_for_user_create(instance_id, serial_users_def)
return serial_users_def
@ -82,7 +82,7 @@ class UserActionsRunner(TestRunner):
queried_user = self.auth_client.users.get(
instance_id, user_name, user_host)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self._assert_user_matches(queried_user, expected_user_def)
def _assert_user_matches(self, user, expected_user_def):
@ -100,7 +100,7 @@ class UserActionsRunner(TestRunner):
def assert_users_list(self, instance_id, expected_user_defs,
expected_http_code, limit=2):
full_list = self.auth_client.users.list(instance_id)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
listed_users = {user.name: user for user in full_list}
self.assert_is_none(full_list.next,
"Unexpected pagination in the list.")
@ -121,7 +121,7 @@ class UserActionsRunner(TestRunner):
# Test list pagination.
list_page = self.auth_client.users.list(instance_id, limit=limit)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self.assert_true(len(list_page) <= limit)
if len(full_list) > limit:
@ -138,7 +138,8 @@ class UserActionsRunner(TestRunner):
"Pagination marker should be the last element "
"in the page.")
list_page = self.auth_client.users.list(instance_id, marker=marker)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code,
client=self.auth_client)
self.assert_pagination_match(
list_page, full_list, limit, len(full_list))
@ -155,7 +156,7 @@ class UserActionsRunner(TestRunner):
user_name, user_host = self._get_user_name_host_pair(user_def)
user_dbs = self.auth_client.users.list_access(instance_id, user_name,
hostname=user_host)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
expected_dbs = {db_def['name'] for db_def in user_def['databases']}
listed_dbs = [db.name for db in user_dbs]
@ -190,7 +191,7 @@ class UserActionsRunner(TestRunner):
database, expected_http_code):
self.auth_client.users.revoke(
instance_id, user_name, database, hostname=user_host)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
user_dbs = self.auth_client.users.list_access(
instance_id, user_name, hostname=user_host)
self.assert_false(any(db.name == database for db in user_dbs),
@ -206,7 +207,7 @@ class UserActionsRunner(TestRunner):
database, expected_http_code):
self.auth_client.users.grant(
instance_id, user_name, [database], hostname=user_host)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
user_dbs = self.auth_client.users.list_access(
instance_id, user_name, hostname=user_host)
self.assert_true(any(db.name == database for db in user_dbs),
@ -339,7 +340,7 @@ class UserActionsRunner(TestRunner):
self.auth_client.users.update_attributes(
instance_id, user_name, update_attribites, user_host)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
# Update the stored definitions with the new value.
expected_def = None
@ -388,7 +389,7 @@ class UserActionsRunner(TestRunner):
user_name, user_host = self._get_user_name_host_pair(user_def)
self.auth_client.users.delete(instance_id, user_name, user_host)
self.assert_client_code(expected_http_code)
self.assert_client_code(expected_http_code, client=self.auth_client)
self._wait_for_user_delete(instance_id, user_name)
def _wait_for_user_delete(self, instance_id, deleted_user_name):