openstack
/
freezer
Code Issues Proposed changes

Fixed a bug in multi level restore and added new --action arg 

- Fixed an issue when restoring multi level backup with queues
- Added th new option --action {backup, restore, info, admin}
- Extended examples in README and other minors log msg changes
- Change homepage in setup.py

Change-Id: I5fa13971a3134d7f001dc3ce5aeabbc95a10626a
This commit is contained in:
Fausto Marzi 2014-11-01 20:55:14 +00:00
parent c1e6e57b7a
commit 983c728433
8 changed files with 156 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
README.rst
View File

@ -23,14 +23,15 @@ Requirements
============
- OpenStack Swift Account (Auth V2 used)
- python >= 2.6 (2.7 advised)
- python
- GNU Tar >= 1.26
- gzip
- OpenSSL
- python-swiftclient >= 2.0.3
- python-keystoneclient >= 0.8.0
- pymongo >= 2.6.2 (if MongoDB backups will be executed)
- At least 128 MB of memory reserved for freezer
- python-swiftclient
- python-keystoneclient
- pymongo
- python-mysqldb
- At least 128 MB of memory reserved for Freezer
Installation & Env Setup
========================
@ -177,7 +178,7 @@ File System Restore:
Execute a file system restore of the backup name
adminui.git::
$ sudo freezerc --container foobar-container-2
$ sudo freezerc --action restore --container foobar-container-2
--backup-name adminui.git
--restore-from-host git-HP-DL380-host-001 --restore-abs-path
/home/git/repositories/adminui.git/
@ -192,7 +193,7 @@ Let's stop mysql service first::
Execute Restore::
$ sudo freezerc --container foobar-container-2
$ sudo freezerc --action restore --container foobar-container-2
--backup-name mysq-prod --restore-from-host db-HP-DL380-host-001
--restore-abs-path /var/lib/mysql --restore-from-date "2014-05-23T23:23:23"
@ -202,10 +203,24 @@ And finally restart mysql::
Execute a MongoDB restore of the backup name mongobigdata::
$ sudo freezerc --container foobar-container-2 --backup-name mongobigdata
$ sudo freezerc --action restore --container foobar-container-2 --backup-name mongobigdata
--restore-from-host db-HP-DL380-host-001 --restore-abs-path
/var/lib/mongo --restore-from-date "2014-05-23T23:23:23"
List remote containers::
$ sudo freezerc --action info -L
List remote objects in container::
$ sudo freezerc --action info --container testcontainer -l
Remove backups older then 1 day::
$ freezerc --action admin --container freezer-dev-test --remove-older-then 1 --backup-name dev-test-01
Architecture
============

8
freezer/arguments.py
View File

@ -34,6 +34,14 @@ def backup_arguments():
a name space called backup_args.
"""
arg_parser = argparse.ArgumentParser(prog='freezerc')
arg_parser.add_argument(
'--action', choices=['backup', 'restore', 'info', 'admin'],
help=(
"Set the action to be taken. backup and restore are"
" self explanatory, info is used to retrieve info from the"
" storage media, while maintenance is used to delete old backups"
" and other admin actions. Default backup."),
dest='action', default='backup')
arg_parser.add_argument(
'-F', '--file-to-backup', action='store',
help="The file or directory you want to back up to Swift",

129
freezer/main.py
View File

@ -1,4 +1,4 @@
'''
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
@ -19,15 +19,15 @@ Hudson (tjh@cryptsoft.com).
========================================================================
Freezer main execution function
'''
"""
from freezer.utils import (
start_time, elapsed_time, set_backup_level, validate_any_args,
start_time, elapsed_time, set_backup_level,
check_backup_existance)
from freezer.swift import (
get_client, get_containers_list, show_containers,
check_container_existance, get_container_content, remove_obj_older_than,
show_objects)
show_objects, create_containers)
from freezer.backup import (
backup_mode_fs, backup_mode_mongo, backup_mode_mysql)
from freezer.restore import restore_fs
@ -36,12 +36,12 @@ import logging
def freezer_main(backup_args):
'''
Program Main Execution. This main function is a wrapper for most
"""Freezer Main execution function.
This main function is a wrapper for most
of the other functions. By calling main() the program execution start
and the respective actions are taken. If you want only use the single
function is probably better to not import main()
'''
"""
# Computing execution start datetime and Timestamp
(time_stamp, today_start) = start_time()
@ -56,52 +56,79 @@ def freezer_main(backup_args):
# Get the list of the containers
backup_args = get_containers_list(backup_args)
if show_containers(backup_args):
elapsed_time(today_start)
return True
# Check if the provided container already exists in swift.
# If it doesn't exist a new one will be created along with the segments
# container as container_segments
backup_args = check_container_existance(backup_args)
# Get the object list of the remote containers and store id in the
# same dict passes as argument under the dict.remote_obj_list namespace
backup_args = get_container_content(backup_args)
if show_objects(backup_args):
elapsed_time(today_start)
return True
# Check if a backup exist in swift with same name. If not, set
# backup level to 0
manifest_meta_dict = check_backup_existance(backup_args)
# Set the right backup level for incremental backup
(backup_args, manifest_meta_dict) = set_backup_level(
backup_args, manifest_meta_dict)
backup_args.manifest_meta_dict = manifest_meta_dict
# File system backup mode selected
if backup_args.mode == 'fs':
# If any of the restore options was specified, then a data restore
# will be executed
if validate_any_args([
backup_args.restore_from_date, backup_args.restore_from_host,
backup_args.restore_abs_path]):
logging.info('[*] Executing FS restore...')
restore_fs(backup_args)
if backup_args.action == 'info' or backup_args.list_container or \
backup_args.list_objects:
if backup_args.list_container:
show_containers(backup_args)
elif backup_args.list_objects:
containers = check_container_existance(backup_args)
if containers['main_container'] is not True:
logging.critical(
'[*] Container {0} not available'.format(
backup_args.container))
return False
backup_args = get_container_content(backup_args)
show_objects(backup_args)
else:
logging.warning(
'[*] No retrieving info options were set. Exiting.')
elapsed_time(today_start)
return False
elapsed_time(today_start)
return True
if backup_args.action == 'restore':
logging.info('[*] Executing FS restore...')
# Check if the provided container already exists in swift.
containers = check_container_existance(backup_args)
if containers['main_container'] is not True:
exc_msg = ('[*] Container: {0} not found. Please provide an '
'existing container.'.format(backup_args.container))
logging.critical(exc_msg)
raise ValueError(exc_msg)
# Get the object list of the remote containers and store it in the
# same dict passes as argument under the dict.remote_obj_list namespace
backup_args = get_container_content(backup_args)
restore_fs(backup_args)
if backup_args.action == 'backup':
# Check if the provided container already exists in swift.
containers = check_container_existance(backup_args)
if containers['main_container'] is not True:
create_containers(backup_args)
# Get the object list of the remote containers and store it in the
# same dict passes as argument under the dict.remote_obj_list namespace
backup_args = get_container_content(backup_args)
# Check if a backup exist in swift with same name. If not, set
# backup level to 0
manifest_meta_dict = check_backup_existance(backup_args)
# Set the right backup level for incremental backup
(backup_args, manifest_meta_dict) = set_backup_level(
backup_args, manifest_meta_dict)
backup_args.manifest_meta_dict = manifest_meta_dict
# File system backup mode selected
if backup_args.mode == 'fs':
backup_mode_fs(backup_args, time_stamp, manifest_meta_dict)
elif backup_args.mode == 'mongo':
backup_mode_mongo(backup_args, time_stamp, manifest_meta_dict)
elif backup_args.mode == 'mysql':
backup_mode_mysql(backup_args, time_stamp, manifest_meta_dict)
else:
logging.critical('[*] Error: Please provide a valid backup mode')
raise ValueError
elif backup_args.mode == 'mongo':
backup_mode_mongo(backup_args, time_stamp, manifest_meta_dict)
elif backup_args.mode == 'mysql':
backup_mode_mysql(backup_args, time_stamp, manifest_meta_dict)
else:
logging.critical('[*] Error: Please provide a valid backup mode')
raise ValueError
remove_obj_older_than(backup_args)
# Admin tasks code should go here, before moving it on a dedicated module
if backup_args.action == 'admin' or backup_args.remove_older_than:
# Remove backups older if set.
remove_obj_older_than(backup_args)
# Elapsed time:
# Compute elapsed time
elapsed_time(today_start)

10
freezer/restore.py
View File

@ -135,15 +135,15 @@ def restore_fs_sort_obj(backup_opt_dict):
# Backups are looped from the last element of the list going
# backwards, as we want to restore starting from the oldest object
write_pipe, read_pipe = Pipe()
for backup in closest_backup_list[::-1]:
write_pipe, read_pipe = Pipe()
process_stream = Process(
target=object_to_stream, args=(
backup_opt_dict, write_pipe, read_pipe, backup,))
process_stream.daemon = True
process_stream.start()
write_pipe.close()
write_pipe.close()
# Start the tar pipe consumer process
tar_stream = Process(
target=tar_restore, args=(backup_opt_dict, read_pipe))
@ -152,3 +152,9 @@ def restore_fs_sort_obj(backup_opt_dict):
read_pipe.close()
process_stream.join()
tar_stream.join()
logging.info(
'[*] Restore execution successfully executed for backup name {0},\
from container {1}, into directory {2}'.format(
backup_opt_dict.backup_name, backup_opt_dict.container,
backup_opt_dict.restore_abs_path))

47
freezer/swift.py
View File

@ -37,6 +37,29 @@ import logging
RESP_CHUNK_SIZE = 65536
def create_containers(backup_opt):
"""Create backup containers
The function is used to create object and segments
containers
:param backup_opt:
:return: True if both containers are succesfully created
"""
# Create backup container
logging.warning(
"[*] Creating container {0}".format(backup_opt.container))
backup_opt.sw_connector.put_container(backup_opt.container)
# Create segments container
logging.warning(
"[*] Creating container segments: {0}".format(
backup_opt.container_segments))
backup_opt.sw_connector.put_container(backup_opt.container_segments)
return True
def show_containers(backup_opt_dict):
"""
Print remote containers in sorted order
@ -179,8 +202,6 @@ def check_container_existance(backup_opt_dict):
"[*] Retrieving container {0}".format(backup_opt_dict.container))
sw_connector = backup_opt_dict.sw_connector
containers_list = sw_connector.get_account()[1]
match_container = None
match_container_seg = None
match_container = [
container_object['name'] for container_object in containers_list
@ -189,29 +210,29 @@ def check_container_existance(backup_opt_dict):
container_object['name'] for container_object in containers_list
if container_object['name'] == backup_opt_dict.container_segments]
# If no container is available, create it and write to logs
# Initialize container dict
containers = {'main_container': False, 'container_segments': False}
if not match_container:
logging.warning("[*] No such container {0} available... ".format(
backup_opt_dict.container))
logging.warning(
"[*] Creating container {0}".format(backup_opt_dict.container))
sw_connector.put_container(backup_opt_dict.container)
else:
logging.info(
"[*] Container {0} found!".format(backup_opt_dict.container))
containers['main_container'] = True
if not match_container_seg:
logging.warning("[*] Creating segments container {0}".format(
backup_opt_dict.container_segments))
sw_connector.put_container(backup_opt_dict.container_segments)
logging.warning(
"[*] No segments container {0} available...".format(
backup_opt_dict.container_segments))
else:
logging.info("[*] Container Segments {0} found!".format(
backup_opt_dict.container_segments))
containers['container_segments'] = True
return backup_opt_dict
return containers
# This function is useless? Remove is and use the single env accordingly
def get_swift_os_env():
"""
Get the swift related environment variable
@ -386,7 +407,7 @@ def object_to_stream(backup_opt_dict, write_pipe, read_pipe, obj_name):
arguments: {0}'.format(','.join(required_list)))
raise ValueError
sw_connector = backup_opt_dict.sw_connector
backup_opt_dict = get_client(backup_opt_dict)
logging.info('[*] Downloading data stream...')
# Close the read pipe in this child as it is unneeded
@ -394,7 +415,7 @@ def object_to_stream(backup_opt_dict, write_pipe, read_pipe, obj_name):
# Chunk size is set by RESP_CHUNK_SIZE and sent to che write
# pipe
read_pipe.close()
for obj_chunk in sw_connector.get_object(
for obj_chunk in backup_opt_dict.sw_connector.get_object(
backup_opt_dict.container, obj_name,
resp_chunk_size=RESP_CHUNK_SIZE)[1]:

8
freezer/tar.py
View File

@ -73,19 +73,13 @@ def tar_restore(backup_opt_dict, read_pipe):
tar_cmd_proc.stdin.write(read_pipe.recv_bytes())
except EOFError:
logging.info(
'[*] Pipe closed as EOF reached. Data transmission terminated.')
'[*] Pipe closed as EOF reached. Data transmitted succesfully.')
tar_err = tar_cmd_proc.communicate()[1]
if 'error' in tar_err.lower():
logging.exception('[*] Restore error: {0}'.format(tar_err))
raise Exception(tar_err)
else:
logging.info(
'[*] Restore execution successfully executed for backup name {0},\
from container {1}, into directory {2}'.format(
backup_opt_dict.backup_name, backup_opt_dict.container,
backup_opt_dict.restore_abs_path))
def tar_incremental(

6
freezer/utils.py
View File

@ -480,9 +480,9 @@ def check_backup_existance(backup_opt_dict):
if not backup_opt_dict.backup_name or not backup_opt_dict.container or \
not backup_opt_dict.remote_obj_list:
logging.warning("[*] A valid Swift container,\
or backup name or container content not available. \
Level 0 backup is being executed ")
logging.warning(
('[*] A valid Swift container, or backup name or container '
'content not available. Level 0 backup is being executed '))
return dict()
logging.info("[*] Retreiving backup name {0} on container \

2
setup.py
View File

@ -33,7 +33,7 @@ def read(*filenames, **kwargs):
setup(
name='freezer',
version='1.0.9-2',
url='http://sourceforge.net/projects/openstack-freezer/',
url='https://github.com/stackforge/freezer',
license='Apache Software License',
author='Fausto Marzi, Ryszard Chojnacki, Emil Dimitrov',
author_email='fausto.marzi@hp.com, ryszard@hp.com, edimitrov@hp.com',