From a853067a13abbc32da4058ba843fd8404042bd44 Mon Sep 17 00:00:00 2001 From: Ruslan Aliev Date: Wed, 21 Dec 2016 17:54:12 +0400 Subject: [PATCH] Block based backup support (rsync) This is a spec for block based support implementation. Using rsync as approach. Change-Id: I38e73ac970b35bceddfcc382b6e533b4629211b6 Signed-off-by: Ruslan Aliev --- README.rst | 1 + requirements.txt | 10 +- .../approved/block_based_backup_support.rst | 229 ++++++++++++++++++ 3 files changed, 235 insertions(+), 5 deletions(-) create mode 100644 specs/ocata/approved/block_based_backup_support.rst diff --git a/README.rst b/README.rst index 94c6ba3..509d880 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,7 @@ Team and repository tags .. image:: http://governance.openstack.org/badges/freezer-specs.svg :target: http://governance.openstack.org/reference/tags/index.html + :remote: .. Change things from this point on diff --git a/requirements.txt b/requirements.txt index 5aaaa59..3d8b678 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -oslosphinx -pbr>=1.6 # Apache-2.0 -sphinx>=1.1.2,<1.2 -testrepository>=0.0.18 -testtools>=0.9.34 +oslosphinx>=4.7.0 # Apache-2.0 +pbr>=1.8 # Apache-2.0 +sphinx>=1.2.1,!=1.3b1,<1.4 # BSD +testrepository>=0.0.18 # Apache-2.0/BSD +testtools>=1.4.0 # MIT yasfb>=0.5.1 diff --git a/specs/ocata/approved/block_based_backup_support.rst b/specs/ocata/approved/block_based_backup_support.rst new file mode 100644 index 0000000..05002d8 --- /dev/null +++ b/specs/ocata/approved/block_based_backup_support.rst @@ -0,0 +1,229 @@ +.. + This work is licensed under a Creative Commons Attribution 3.0 Unported + License. + + http://creativecommons.org/licenses/by/3.0/legalcode + +================================== +Block based backup support (rsync) +================================== + +https://blueprints.launchpad.net/freezer/+spec/rsync + +Taking advantage of the rsync to provide a possibility to create +space/bandwidth efficient backups. + +Problem description +=================== + +Currently Freezer checks only ctime and mtime inode information +to verify if files are changed or not (tar functionality). While +this approach gives speed (time efficient), it is not bandwidth +and storage efficient. Freezer needs to support both rsync and tar +approach to execute incremental backups and restore. + +Since Freezer will provide two options for incremental backups, it +would be more convenient to choose the best approach to backup data +in accordance with each particular case (more speed or storage/bandwidth +efficient). + +Use Cases +--------- + +* For developers, this change will not create negative impacts because + this code will be gracefully bundled in Freezer engine API and + will not cause any major changes in Freezer architecture. + +* For Deployers there is no need to install any additional components, + Freezer will use it's own implementation of rsync algorithm + (written in Python). + +* For End User it would be less difficult to select more efficient + option for create backups based on dataset (e.g. few big files or a lot of + small files) for backup and speed/storage/bandwidth requirements, + since Freezer would support both rsync and tar approaches. + +Proposed change +=============== + +Implementing the new engine classes for rsync (as well as for tar). +Providing new engine (-e) choice in config. + +For this type of backup will be created following metadata structure: + +files_meta = { + 'files': {}, + 'directories': {}, + 'meta': { + 'broken_links_tot': '', + 'total_files': '', + 'total_directories': '', + 'backup_size_on_disk': '', + 'backup_size_uncompressed': '', + 'backup_size_compressed': '', + 'platform': sys.platform + }, + 'abs_backup_path': os.getcwd(), + 'broken_links': [], + 'rsync_struct_ver': RSYNC_DATA_STRUCT_VERSION, + 'rsync_block_size': RSYNC_BLOCK_SIZE} + + +file_meta = {'inode': { + 'inumber': os_stat.st_ino, + 'nlink': os_stat.st_nlink, + 'mode': file_mode, + 'uid': os_stat.st_uid, + 'gid': os_stat.st_gid, + 'size': os_stat.st_size, + 'devmajor': os.major(dev), + 'devminor': os.minor(dev), + 'mtime': mtime, + 'ctime': ctime, + 'uname': uname, + 'gname': gname, + 'ftype': file_type, + 'lname': lname, + 'rsync_block_size': rsync_block_size, + 'file_status: status + } + } + +Current version of implementation you always can find here [1]. + +Alternatives +------------ + +Because of the flexibility, speed, and scriptability of rsync, it has +become a standard Linux utility, included in all popular Linux distributions. +It has been ported to Windows (via Cygwin, Grsync, or SFU), FreeBSD, NetBSD, +OpenBSD, and Mac OS. De facto, rsync is the default fallback for most data +transfers. It has a clear algorithm written for 20 years ago and different +variations (e.g. acrosync, zsync, etc). librsync is used by Dropbox. + +Using other alternative (like bbcp or lftp) would not be more effective +or portable solution. + +Data model impact +----------------- + +Changes in data model has already described in oslo.db migration document. +Actions entity should contain 'engine' field for performing appropriate action +using particular type of engine (tar, rsync or openstack). + +From new relational database schema: + +Actions + action_id (uuid) [p_key] + resource (varchar) + type (varchar) + name (varchar) + application (varchar) + engine (varchar) <-- Require this + snapshot (varchar) + storage (varchar) + global_options (JSON) + application_options (JSON) + storage_options (JSON) + snapshot_options (JSON) + engine_options (JSON) + + +REST API impact +--------------- + +None. + +Security impact +--------------- + +None. + +Notifications impact +-------------------- + +There are no special logs will be added, just some info messages about +start/stop backup process, backup metrics, etc. + +Other end user impact +--------------------- + +* There are no additional changes to python-freezerclient CLI. To choice + appropriate engine for action, end user should specify 'engine' field + in provided JSON configuration in case of creating or updating action. + +* freezer-web-ui should provide additional 'engine' field in 'Action + Configuration' window. It has to be drop-down list with values 'tar', + 'rsync' or 'openstack'. + +Performance Impact +------------------ + +None. + +Other deployer impact +--------------------- + +Will be added new choice to freezer-agent -e (engine) option - 'rsync'. + +Developer impact +---------------- + +None. + +Implementation +============== + +Assignee(s) +----------- + +Primary assignee: + Ruslan Aliev (raliev) + +Other contributors: + Fausto Marzi (daemontool) + +Work Items +---------- + +* implementing the new engine (rsync) + +* bundling this engine to freezer code (API calls) and mechanism + for using this engine ('-e rsync' option) + +* implementing the new database schema for actions (oslo.db migration) + +* updating freezer-web-ui 'Action Configuration' window + +* updating documentation + +Dependencies +============ + +* This spec depends on Freezer oslo.db migration [2]. + +* Pluggable engines described here [3]. + +* There are no additional library dependencies. + +Testing +======= + +There is a question - do we actually need separate tempest test +for this change or we can be satisfied with existing one? + +Documentation Impact +==================== + +* freezer README doc + +* freezer-api README doc + +* freezer-web-ui README doc + +References +========== + +.. [1] https://review.openstack.org/#/c/409796/ +.. [2] https://etherpad.openstack.org/p/freezer_mysql_migration +.. [3] https://etherpad.openstack.org/p/freezer_new_archi