Implement scheduled backups

Implements Trove client APIs to implement scheduled backups
via Mistral workflow.

Change-Id: I012eb88359e063adbb86979a8fbd2e2a1e83f816
Implements: blueprint scheduled-backups
This commit is contained in:
Morgan Jones 2016-06-13 15:26:08 -04:00
parent f5616b7d52
commit 599171ade4
5 changed files with 390 additions and 0 deletions

View File

@ -0,0 +1,3 @@
features:
- Implements trove schedule-* and execution-* commands to support
scheduled backups.

View File

@ -11,3 +11,4 @@ Babel>=2.3.4 # BSD
keystoneauth1>=2.10.0 # Apache-2.0
six>=1.9.0 # MIT
python-swiftclient>=2.2.0 # Apache-2.0
python-mistralclient>=2.0.0 # Apache-2.0

View File

@ -14,6 +14,7 @@
# under the License.
import mock
from mock import patch
import testtools
import uuid
@ -147,3 +148,125 @@ class BackupManagerTest(testtools.TestCase):
resp.status_code = 422
self.backups.api.client.delete = mock.Mock(return_value=(resp, None))
self.assertRaises(Exception, self.backups.delete, 'backup1')
@patch('troveclient.v1.backups.mistral_client')
def test_auth_mistral_client(self, mistral_client):
with patch.object(self.backups.api.client, 'auth') as auth:
self.backups._get_mistral_client()
mistral_client.assert_called_with(
auth_url=auth.auth_url, username=auth._username,
api_key=auth._password,
project_name=auth._project_name)
def test_build_schedule(self):
cron_trigger = mock.Mock()
wf_input = {'name': 'foo', 'instance': 'myinst', 'parent_id': None}
sched = self.backups._build_schedule(cron_trigger, wf_input)
self.assertEqual(cron_trigger.name, sched.id)
self.assertEqual(wf_input['name'], sched.name)
self.assertEqual(wf_input['instance'], sched.instance)
self.assertEqual(cron_trigger.workflow_input, sched.input)
def test_schedule_create(self):
instance = mock.Mock()
pattern = mock.Mock()
name = 'myback'
def make_cron_trigger(name, wf, workflow_input=None, pattern=None):
return mock.Mock(name=name, pattern=pattern,
workflow_input=workflow_input)
cron_triggers = mock.Mock()
cron_triggers.create = mock.Mock(side_effect=make_cron_trigger)
mistral_client = mock.Mock(cron_triggers=cron_triggers)
sched = self.backups.schedule_create(instance, pattern, name,
mistral_client=mistral_client)
self.assertEqual(pattern, sched.pattern)
self.assertEqual(name, sched.name)
self.assertEqual(instance.id, sched.instance)
def test_schedule_list(self):
instance = mock.Mock(id='the_uuid')
backup_name = "wf2"
test_input = [('wf1', 'foo'), (backup_name, instance.id)]
cron_triggers = mock.Mock()
cron_triggers.list = mock.Mock(
return_value=[
mock.Mock(workflow_input='{"name": "%s", "instance": "%s"}'
% (name, inst), name=name)
for name, inst in test_input
])
mistral_client = mock.Mock(cron_triggers=cron_triggers)
sched_list = self.backups.schedule_list(instance, mistral_client)
self.assertEqual(1, len(sched_list))
the_sched = sched_list.pop()
self.assertEqual(backup_name, the_sched.name)
self.assertEqual(instance.id, the_sched.instance)
def test_schedule_show(self):
instance = mock.Mock(id='the_uuid')
backup_name = "myback"
cron_triggers = mock.Mock()
cron_triggers.get = mock.Mock(
return_value=mock.Mock(
name=backup_name,
workflow_input='{"name": "%s", "instance": "%s"}'
% (backup_name, instance.id)))
mistral_client = mock.Mock(cron_triggers=cron_triggers)
sched = self.backups.schedule_show("dummy", mistral_client)
self.assertEqual(backup_name, sched.name)
self.assertEqual(instance.id, sched.instance)
def test_schedule_delete(self):
cron_triggers = mock.Mock()
cron_triggers.delete = mock.Mock()
mistral_client = mock.Mock(cron_triggers=cron_triggers)
self.backups.schedule_delete("dummy", mistral_client)
cron_triggers.delete.assert_called()
def test_execution_list(self):
instance = mock.Mock(id='the_uuid')
wf_input = '{"name": "wf2", "instance": "%s"}' % instance.id
wf_name = self.backups.backup_create_workflow
execution_list_result = [
[mock.Mock(id=1, input=wf_input, workflow_name=wf_name,
to_dict=mock.Mock(return_value={'id': 1})),
mock.Mock(id=2, input="{}", workflow_name=wf_name)],
[mock.Mock(id=3, input=wf_input, workflow_name=wf_name,
to_dict=mock.Mock(return_value={'id': 3})),
mock.Mock(id=4, input="{}", workflow_name=wf_name)],
[mock.Mock(id=5, input=wf_input, workflow_name=wf_name,
to_dict=mock.Mock(return_value={'id': 5})),
mock.Mock(id=6, input="{}", workflow_name=wf_name)],
[mock.Mock(id=7, input=wf_input, workflow_name="bar"),
mock.Mock(id=8, input="{}", workflow_name=wf_name)]
]
cron_triggers = mock.Mock()
cron_triggers.get = mock.Mock(
return_value=mock.Mock(workflow_name=wf_name,
workflow_input=wf_input))
mistral_executions = mock.Mock()
mistral_executions.list = mock.Mock(side_effect=execution_list_result)
mistral_client = mock.Mock(cron_triggers=cron_triggers,
executions=mistral_executions)
el = self.backups.execution_list("dummy", mistral_client, limit=2)
self.assertEqual(2, len(el))
el = self.backups.execution_list("dummy", mistral_client, limit=2)
self.assertEqual(1, len(el))
the_exec = el.pop()
self.assertEqual(5, the_exec.id)
def test_execution_delete(self):
mistral_executions = mock.Mock()
mistral_executions.delete = mock.Mock()
mistral_client = mock.Mock(executions=mistral_executions)
self.backups.execution_delete("dummy", mistral_client)
mistral_executions.delete.assert_called()

View File

@ -15,6 +15,12 @@
# License for the specific language governing permissions and limitations
# under the License.
import json
import six
import uuid
from mistralclient.api.client import client as mistral_client
from troveclient import base
from troveclient import common
@ -25,6 +31,21 @@ class Backup(base.Resource):
return "<Backup: %s>" % self.name
class Schedule(base.Resource):
"""Schedule is a resource used to hold information about scheduled backups.
"""
def __repr__(self):
return "<Schedule: %s>" % self.name
class ScheduleExecution(base.Resource):
"""ScheduleExecution is a resource used to hold information about
the execution of a scheduled backup.
"""
def __repr__(self):
return "<Execution: %s>" % self.name
class Backups(base.ManagerWithFind):
"""Manage :class:`Backups` information."""
@ -87,3 +108,171 @@ class Backups(base.ManagerWithFind):
url = "/backups/%s" % base.getid(backup)
resp, body = self.api.client.delete(url)
common.check_for_exceptions(resp, body, url)
backup_create_workflow = "trove.backup_create"
def _get_mistral_client(self):
if hasattr(self.api.client, 'auth'):
auth_url = self.api.client.auth.auth_url
user = self.api.client.auth._username
key = self.api.client.auth._password
tenant_name = self.api.client.auth._project_name
else:
auth_url = self.api.client.auth_url
user = self.api.client.username
key = self.api.client.password
tenant_name = self.api.client.tenant
return mistral_client(auth_url=auth_url, username=user, api_key=key,
project_name=tenant_name)
def _build_schedule(self, cron_trigger, wf_input):
if isinstance(wf_input, six.string_types):
wf_input = json.loads(wf_input)
sched_info = {"id": cron_trigger.name,
"name": wf_input["name"],
"instance": wf_input['instance'],
"parent_id": wf_input.get('parent_id', None),
"created_at": cron_trigger.created_at,
"next_execution_time": cron_trigger.next_execution_time,
"pattern": cron_trigger.pattern,
"input": cron_trigger.workflow_input
}
if hasattr(cron_trigger, 'updated_at'):
sched_info["updated_at"] = cron_trigger.updated_at
return Schedule(self, sched_info, loaded=True)
def schedule_create(self, instance, pattern, name,
description=None, incremental=None,
mistral_client=None):
"""Create a new schedule to backup the given instance.
:param instance: instance to backup.
:param: pattern: cron pattern for schedule.
:param name: name for backup.
:param description: (optional).
:param incremental: flag for incremental backup (optional).
:returns: :class:`Backups`
"""
if not mistral_client:
mistral_client = self._get_mistral_client()
inst_id = base.getid(instance)
cron_name = str(uuid.uuid4())
wf_input = {"instance": inst_id,
"name": name,
"description": description,
"incremental": incremental
}
cron_trigger = mistral_client.cron_triggers.create(
cron_name, self.backup_create_workflow, pattern=pattern,
workflow_input=wf_input)
return self._build_schedule(cron_trigger, wf_input)
def schedule_list(self, instance, mistral_client=None):
"""Get a list of all backup schedules for an instance.
:param: instance for which to list schedules.
:rtype: list of :class:`Schedule`.
"""
inst_id = base.getid(instance)
if not mistral_client:
mistral_client = self._get_mistral_client()
return [self._build_schedule(cron_trig, cron_trig.workflow_input)
for cron_trig in mistral_client.cron_triggers.list()
if inst_id in cron_trig.workflow_input]
def schedule_show(self, schedule, mistral_client=None):
"""Get details of a backup schedule.
:param: schedule to show.
:rtype: :class:`Schedule`.
"""
if isinstance(schedule, Schedule):
schedule = schedule.id
if not mistral_client:
mistral_client = self._get_mistral_client()
schedule = mistral_client.cron_triggers.get(schedule)
return self._build_schedule(schedule, schedule.workflow_input)
def schedule_delete(self, schedule, mistral_client=None):
"""Remove a given backup schedule.
:param schedule: schedule to delete.
"""
if isinstance(schedule, Schedule):
schedule = schedule.id
if not mistral_client:
mistral_client = self._get_mistral_client()
mistral_client.cron_triggers.delete(schedule)
def execution_list(self, schedule, mistral_client=None,
marker='', limit=None):
"""Get a list of all executions of a scheduled backup.
:param: schedule for which to list executions.
:rtype: list of :class:`ScheduleExecution`.
"""
if isinstance(schedule, Schedule):
schedule = schedule.id
if isinstance(marker, ScheduleExecution):
marker = getattr(marker, 'id')
if not mistral_client:
mistral_client = self._get_mistral_client()
cron_trigger = mistral_client.cron_triggers.get(schedule)
ct_input = json.loads(cron_trigger.workflow_input)
def mistral_execution_generator():
m = marker
while True:
the_list = mistral_client.executions.list(marker=m, limit=50,
sort_dirs='desc')
if the_list:
for the_item in the_list:
yield the_item
m = the_list[-1].id
else:
raise StopIteration()
def execution_list_generator():
yielded = 0
for sexec in mistral_execution_generator():
if (sexec.workflow_name == cron_trigger.workflow_name
and ct_input == json.loads(sexec.input)):
yield ScheduleExecution(self, sexec.to_dict(),
loaded=True)
yielded += 1
if limit and yielded == limit:
raise StopIteration()
return list(execution_list_generator())
def execution_delete(self, execution, mistral_client=None):
"""Remove a given schedule execution.
:param id: id of execution to remove.
"""
exec_id = (execution.id if isinstance(execution, ScheduleExecution)
else execution)
if isinstance(execution, ScheduleExecution):
execution = execution.name
if not mistral_client:
mistral_client = self._get_mistral_client()
mistral_client.executions.delete(exec_id)

View File

@ -982,6 +982,80 @@ def do_backup_copy(cs, args):
_print_object(backup)
@utils.arg('instance', metavar='<instance>',
help='ID or name of the instance.')
@utils.arg('pattern', metavar='<pattern>',
help='Cron style pattern describing schedule occurrence.')
@utils.arg('name', metavar='<name>', help='Name of the backup.')
@utils.arg('--description', metavar='<description>',
default=None,
help='An optional description for the backup.')
@utils.arg('--incremental', action="store_true", default=False,
help='Flag to select incremental backup based on most recent'
' backup.')
@utils.service_type('database')
def do_schedule_create(cs, args):
"""Schedules backups for an instance."""
instance = _find_instance(cs, args.instance)
backup = cs.backups.schedule_create(instance, args.pattern, args.name,
description=args.description,
incremental=args.incremental)
_print_object(backup)
@utils.arg('instance', metavar='<instance>',
help='ID or name of the instance.')
@utils.service_type('database')
def do_schedule_list(cs, args):
"""Lists scheduled backups for an instance."""
instance = _find_instance(cs, args.instance)
schedules = cs.backups.schedule_list(instance)
utils.print_list(schedules, ['id', 'name', 'pattern',
'next_execution_time'],
order_by='next_execution_time')
@utils.arg('id', metavar='<schedule id>', help='Id of the schedule.')
@utils.service_type('database')
def do_schedule_show(cs, args):
"""Shows details of a schedule."""
_print_object(cs.backups.schedule_show(args.id))
@utils.arg('id', metavar='<schedule id>', help='Id of the schedule.')
@utils.service_type('database')
def do_schedule_delete(cs, args):
"""Deletes a schedule."""
cs.backups.schedule_delete(args.id)
@utils.arg('id', metavar='<schedule id>', help='Id of the schedule.')
@utils.arg('--limit', metavar='<limit>',
default=None, type=int,
help='Return up to N number of the most recent executions.')
@utils.arg('--marker', metavar='<ID>', type=str, default=None,
help='Begin displaying the results for IDs greater than the '
'specified marker. When used with --limit, set this to '
'the last ID displayed in the previous run.')
@utils.service_type('database')
def do_execution_list(cs, args):
"""Lists executions of a scheduled backup of an instance."""
executions = cs.backups.execution_list(args.id, marker=args.marker,
limit=args.limit)
utils.print_list(executions, ['id', 'created_at', 'state', 'output'],
labels={'created_at': 'Execution Time'},
order_by='created_at')
@utils.arg('execution', metavar='<execution>',
help='Id of the execution to delete.')
@utils.service_type('database')
def do_execution_delete(cs, args):
"""Deletes an execution."""
cs.backups.execution_delete(args.execution)
# Database related actions
@utils.arg('instance', metavar='<instance>',