diff --git a/README.rst b/README.rst index b8bc315d..62e1e8ef 100644 --- a/README.rst +++ b/README.rst @@ -23,14 +23,15 @@ Requirements ============ - OpenStack Swift Account (Auth V2 used) -- python >= 2.6 (2.7 advised) +- python - GNU Tar >= 1.26 - gzip - OpenSSL -- python-swiftclient >= 2.0.3 -- python-keystoneclient >= 0.8.0 -- pymongo >= 2.6.2 (if MongoDB backups will be executed) -- At least 128 MB of memory reserved for freezer +- python-swiftclient +- python-keystoneclient +- pymongo +- python-mysqldb +- At least 128 MB of memory reserved for Freezer Installation & Env Setup ======================== @@ -177,7 +178,7 @@ File System Restore: Execute a file system restore of the backup name adminui.git:: - $ sudo freezerc --container foobar-container-2 + $ sudo freezerc --action restore --container foobar-container-2 --backup-name adminui.git --restore-from-host git-HP-DL380-host-001 --restore-abs-path /home/git/repositories/adminui.git/ @@ -192,7 +193,7 @@ Let's stop mysql service first:: Execute Restore:: - $ sudo freezerc --container foobar-container-2 + $ sudo freezerc --action restore --container foobar-container-2 --backup-name mysq-prod --restore-from-host db-HP-DL380-host-001 --restore-abs-path /var/lib/mysql --restore-from-date "2014-05-23T23:23:23" @@ -202,10 +203,24 @@ And finally restart mysql:: Execute a MongoDB restore of the backup name mongobigdata:: - $ sudo freezerc --container foobar-container-2 --backup-name mongobigdata + $ sudo freezerc --action restore --container foobar-container-2 --backup-name mongobigdata --restore-from-host db-HP-DL380-host-001 --restore-abs-path /var/lib/mongo --restore-from-date "2014-05-23T23:23:23" + +List remote containers:: + + $ sudo freezerc --action info -L + +List remote objects in container:: + + $ sudo freezerc --action info --container testcontainer -l + + +Remove backups older then 1 day:: + + $ freezerc --action admin --container freezer-dev-test --remove-older-then 1 --backup-name dev-test-01 + Architecture ============ diff --git a/freezer/arguments.py b/freezer/arguments.py index b23cd500..3f9f07c8 100644 --- a/freezer/arguments.py +++ b/freezer/arguments.py @@ -34,6 +34,14 @@ def backup_arguments(): a name space called backup_args. """ arg_parser = argparse.ArgumentParser(prog='freezerc') + arg_parser.add_argument( + '--action', choices=['backup', 'restore', 'info', 'admin'], + help=( + "Set the action to be taken. backup and restore are" + " self explanatory, info is used to retrieve info from the" + " storage media, while maintenance is used to delete old backups" + " and other admin actions. Default backup."), + dest='action', default='backup') arg_parser.add_argument( '-F', '--file-to-backup', action='store', help="The file or directory you want to back up to Swift", diff --git a/freezer/main.py b/freezer/main.py index 10253024..f6022d6d 100644 --- a/freezer/main.py +++ b/freezer/main.py @@ -1,4 +1,4 @@ -''' +""" Copyright 2014 Hewlett-Packard Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,15 +19,15 @@ Hudson (tjh@cryptsoft.com). ======================================================================== Freezer main execution function -''' +""" from freezer.utils import ( - start_time, elapsed_time, set_backup_level, validate_any_args, + start_time, elapsed_time, set_backup_level, check_backup_existance) from freezer.swift import ( get_client, get_containers_list, show_containers, check_container_existance, get_container_content, remove_obj_older_than, - show_objects) + show_objects, create_containers) from freezer.backup import ( backup_mode_fs, backup_mode_mongo, backup_mode_mysql) from freezer.restore import restore_fs @@ -36,12 +36,12 @@ import logging def freezer_main(backup_args): - ''' - Program Main Execution. This main function is a wrapper for most + """Freezer Main execution function. + This main function is a wrapper for most of the other functions. By calling main() the program execution start and the respective actions are taken. If you want only use the single function is probably better to not import main() - ''' + """ # Computing execution start datetime and Timestamp (time_stamp, today_start) = start_time() @@ -56,52 +56,79 @@ def freezer_main(backup_args): # Get the list of the containers backup_args = get_containers_list(backup_args) - if show_containers(backup_args): - elapsed_time(today_start) - return True - - # Check if the provided container already exists in swift. - # If it doesn't exist a new one will be created along with the segments - # container as container_segments - backup_args = check_container_existance(backup_args) - - # Get the object list of the remote containers and store id in the - # same dict passes as argument under the dict.remote_obj_list namespace - backup_args = get_container_content(backup_args) - - if show_objects(backup_args): - elapsed_time(today_start) - return True - - # Check if a backup exist in swift with same name. If not, set - # backup level to 0 - manifest_meta_dict = check_backup_existance(backup_args) - - # Set the right backup level for incremental backup - (backup_args, manifest_meta_dict) = set_backup_level( - backup_args, manifest_meta_dict) - - backup_args.manifest_meta_dict = manifest_meta_dict - # File system backup mode selected - if backup_args.mode == 'fs': - # If any of the restore options was specified, then a data restore - # will be executed - if validate_any_args([ - backup_args.restore_from_date, backup_args.restore_from_host, - backup_args.restore_abs_path]): - logging.info('[*] Executing FS restore...') - restore_fs(backup_args) + if backup_args.action == 'info' or backup_args.list_container or \ + backup_args.list_objects: + if backup_args.list_container: + show_containers(backup_args) + elif backup_args.list_objects: + containers = check_container_existance(backup_args) + if containers['main_container'] is not True: + logging.critical( + '[*] Container {0} not available'.format( + backup_args.container)) + return False + backup_args = get_container_content(backup_args) + show_objects(backup_args) else: + logging.warning( + '[*] No retrieving info options were set. Exiting.') + elapsed_time(today_start) + return False + + elapsed_time(today_start) + return True + + if backup_args.action == 'restore': + logging.info('[*] Executing FS restore...') + + # Check if the provided container already exists in swift. + containers = check_container_existance(backup_args) + if containers['main_container'] is not True: + exc_msg = ('[*] Container: {0} not found. Please provide an ' + 'existing container.'.format(backup_args.container)) + logging.critical(exc_msg) + raise ValueError(exc_msg) + + # Get the object list of the remote containers and store it in the + # same dict passes as argument under the dict.remote_obj_list namespace + backup_args = get_container_content(backup_args) + restore_fs(backup_args) + + if backup_args.action == 'backup': + # Check if the provided container already exists in swift. + containers = check_container_existance(backup_args) + + if containers['main_container'] is not True: + create_containers(backup_args) + + # Get the object list of the remote containers and store it in the + # same dict passes as argument under the dict.remote_obj_list namespace + backup_args = get_container_content(backup_args) + + # Check if a backup exist in swift with same name. If not, set + # backup level to 0 + manifest_meta_dict = check_backup_existance(backup_args) + + # Set the right backup level for incremental backup + (backup_args, manifest_meta_dict) = set_backup_level( + backup_args, manifest_meta_dict) + + backup_args.manifest_meta_dict = manifest_meta_dict + # File system backup mode selected + if backup_args.mode == 'fs': backup_mode_fs(backup_args, time_stamp, manifest_meta_dict) - elif backup_args.mode == 'mongo': - backup_mode_mongo(backup_args, time_stamp, manifest_meta_dict) - elif backup_args.mode == 'mysql': - backup_mode_mysql(backup_args, time_stamp, manifest_meta_dict) - else: - logging.critical('[*] Error: Please provide a valid backup mode') - raise ValueError + elif backup_args.mode == 'mongo': + backup_mode_mongo(backup_args, time_stamp, manifest_meta_dict) + elif backup_args.mode == 'mysql': + backup_mode_mysql(backup_args, time_stamp, manifest_meta_dict) + else: + logging.critical('[*] Error: Please provide a valid backup mode') + raise ValueError - remove_obj_older_than(backup_args) + # Admin tasks code should go here, before moving it on a dedicated module + if backup_args.action == 'admin' or backup_args.remove_older_than: + # Remove backups older if set. + remove_obj_older_than(backup_args) - # Elapsed time: + # Compute elapsed time elapsed_time(today_start) diff --git a/freezer/restore.py b/freezer/restore.py index a13d6a34..df5fb255 100644 --- a/freezer/restore.py +++ b/freezer/restore.py @@ -135,15 +135,15 @@ def restore_fs_sort_obj(backup_opt_dict): # Backups are looped from the last element of the list going # backwards, as we want to restore starting from the oldest object - write_pipe, read_pipe = Pipe() for backup in closest_backup_list[::-1]: + write_pipe, read_pipe = Pipe() process_stream = Process( target=object_to_stream, args=( backup_opt_dict, write_pipe, read_pipe, backup,)) process_stream.daemon = True process_stream.start() - write_pipe.close() + write_pipe.close() # Start the tar pipe consumer process tar_stream = Process( target=tar_restore, args=(backup_opt_dict, read_pipe)) @@ -152,3 +152,9 @@ def restore_fs_sort_obj(backup_opt_dict): read_pipe.close() process_stream.join() tar_stream.join() + + logging.info( + '[*] Restore execution successfully executed for backup name {0},\ + from container {1}, into directory {2}'.format( + backup_opt_dict.backup_name, backup_opt_dict.container, + backup_opt_dict.restore_abs_path)) diff --git a/freezer/swift.py b/freezer/swift.py index 83fc39db..65bc02a0 100644 --- a/freezer/swift.py +++ b/freezer/swift.py @@ -37,6 +37,29 @@ import logging RESP_CHUNK_SIZE = 65536 +def create_containers(backup_opt): + """Create backup containers + The function is used to create object and segments + containers + + :param backup_opt: + :return: True if both containers are succesfully created + """ + + # Create backup container + logging.warning( + "[*] Creating container {0}".format(backup_opt.container)) + backup_opt.sw_connector.put_container(backup_opt.container) + + # Create segments container + logging.warning( + "[*] Creating container segments: {0}".format( + backup_opt.container_segments)) + backup_opt.sw_connector.put_container(backup_opt.container_segments) + + return True + + def show_containers(backup_opt_dict): """ Print remote containers in sorted order @@ -179,8 +202,6 @@ def check_container_existance(backup_opt_dict): "[*] Retrieving container {0}".format(backup_opt_dict.container)) sw_connector = backup_opt_dict.sw_connector containers_list = sw_connector.get_account()[1] - match_container = None - match_container_seg = None match_container = [ container_object['name'] for container_object in containers_list @@ -189,29 +210,29 @@ def check_container_existance(backup_opt_dict): container_object['name'] for container_object in containers_list if container_object['name'] == backup_opt_dict.container_segments] - # If no container is available, create it and write to logs + # Initialize container dict + containers = {'main_container': False, 'container_segments': False} + if not match_container: logging.warning("[*] No such container {0} available... ".format( backup_opt_dict.container)) - logging.warning( - "[*] Creating container {0}".format(backup_opt_dict.container)) - sw_connector.put_container(backup_opt_dict.container) else: logging.info( "[*] Container {0} found!".format(backup_opt_dict.container)) + containers['main_container'] = True if not match_container_seg: - logging.warning("[*] Creating segments container {0}".format( - backup_opt_dict.container_segments)) - sw_connector.put_container(backup_opt_dict.container_segments) + logging.warning( + "[*] No segments container {0} available...".format( + backup_opt_dict.container_segments)) else: logging.info("[*] Container Segments {0} found!".format( backup_opt_dict.container_segments)) + containers['container_segments'] = True - return backup_opt_dict + return containers -# This function is useless? Remove is and use the single env accordingly def get_swift_os_env(): """ Get the swift related environment variable @@ -386,7 +407,7 @@ def object_to_stream(backup_opt_dict, write_pipe, read_pipe, obj_name): arguments: {0}'.format(','.join(required_list))) raise ValueError - sw_connector = backup_opt_dict.sw_connector + backup_opt_dict = get_client(backup_opt_dict) logging.info('[*] Downloading data stream...') # Close the read pipe in this child as it is unneeded @@ -394,7 +415,7 @@ def object_to_stream(backup_opt_dict, write_pipe, read_pipe, obj_name): # Chunk size is set by RESP_CHUNK_SIZE and sent to che write # pipe read_pipe.close() - for obj_chunk in sw_connector.get_object( + for obj_chunk in backup_opt_dict.sw_connector.get_object( backup_opt_dict.container, obj_name, resp_chunk_size=RESP_CHUNK_SIZE)[1]: diff --git a/freezer/tar.py b/freezer/tar.py index c6893392..d16cbf5a 100644 --- a/freezer/tar.py +++ b/freezer/tar.py @@ -73,19 +73,13 @@ def tar_restore(backup_opt_dict, read_pipe): tar_cmd_proc.stdin.write(read_pipe.recv_bytes()) except EOFError: logging.info( - '[*] Pipe closed as EOF reached. Data transmission terminated.') + '[*] Pipe closed as EOF reached. Data transmitted succesfully.') tar_err = tar_cmd_proc.communicate()[1] if 'error' in tar_err.lower(): logging.exception('[*] Restore error: {0}'.format(tar_err)) raise Exception(tar_err) - else: - logging.info( - '[*] Restore execution successfully executed for backup name {0},\ - from container {1}, into directory {2}'.format( - backup_opt_dict.backup_name, backup_opt_dict.container, - backup_opt_dict.restore_abs_path)) def tar_incremental( diff --git a/freezer/utils.py b/freezer/utils.py index de1999d7..e9b2d433 100644 --- a/freezer/utils.py +++ b/freezer/utils.py @@ -480,9 +480,9 @@ def check_backup_existance(backup_opt_dict): if not backup_opt_dict.backup_name or not backup_opt_dict.container or \ not backup_opt_dict.remote_obj_list: - logging.warning("[*] A valid Swift container,\ - or backup name or container content not available. \ - Level 0 backup is being executed ") + logging.warning( + ('[*] A valid Swift container, or backup name or container ' + 'content not available. Level 0 backup is being executed ')) return dict() logging.info("[*] Retreiving backup name {0} on container \ diff --git a/setup.py b/setup.py index be32b8a5..c4e10d4f 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ def read(*filenames, **kwargs): setup( name='freezer', version='1.0.9-2', - url='http://sourceforge.net/projects/openstack-freezer/', + url='https://github.com/stackforge/freezer', license='Apache Software License', author='Fausto Marzi, Ryszard Chojnacki, Emil Dimitrov', author_email='fausto.marzi@hp.com, ryszard@hp.com, edimitrov@hp.com',