diff --git a/README.rst b/README.rst index 015b6e56..1fdbf443 100644 --- a/README.rst +++ b/README.rst @@ -348,6 +348,7 @@ Freezer can use: --storage ssh --ssh-username ubuntu --ssh-key ~/.ssh/id_rsa --ssh-host 8.8.8.8 +**Note** ssh keys with passphrase are not supported at the moment. Restore ------- @@ -400,9 +401,9 @@ List remote objects in container:: $ sudo freezerc --action info --container freezer_testcontainer -l -Remove backups older then 1 day:: +Remove backups older than a date:: - $ freezerc --action admin --container freezer_dev-test --remove-older-then 1 --backup-name dev-test-01 + $ freezer-agent --action admin --container freezer_dev-test --remove-before-date 2016-07-11T00:00:00 --backup-name dev-test-01 Cinder restore currently creates a volume with the contents of the saved one, but @@ -444,8 +445,9 @@ Freezer architectural components are the following: Freezer uses GNU Tar under the hood to execute incremental backup and restore. When a key is provided, it uses OpenSSL to encrypt data. (AES-256-CFB) -============= -Freezer architecture is composed by the following components: + +Freezer components. +------------------- +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ | Component | Description | @@ -987,3 +989,92 @@ optional arguments: Remote username for ssh storage only --ssh-host SSH_HOST Remote host for ssh storage only --ssh-port SSH_PORT Remote port for ssh storage only (default 22) +======= + +Scheduler Options +----------------- + +To get an updated sample of freezer-scheduler configuration you the following command:: + + oslo-config-generator --config-file config-generator/scheduler.conf + +you will find the update sample file in etc/scheduler.conf.sample + +Agent Options +------------- + +To list options available in freezer-agent use the following command:: + + oslo-config-generator --namespace freezer --namespace oslo.log + +this will print all options to the screen you direct the output to a file if you want:: + + oslo-config-generator --namespace freezer --namespace oslo.log --output-file etc/agent.conf.sample + + +Bandwidth limitation (Trickle) +------------------------------ + +Trickle for bandwidth limiting ( How it works ): +We have 3 cases to handle +1- User used --upload-limit or --download-limit from the cli +We need to remove these limits from the cli arguments and then run trickle +using subprocess + +EX:: + + # freezer-agent --action backup -F /etc/ -C freezer --upload-limit = 1k + +this will be translated to:: + + # trickle -u 1024 -d -1 freezer-agent --action backup -F /etc/ -C freezer + +2- User used config files to execute an action + +We need to create a new config file without the limits So we will get the all +the arguments provided and remove limits then run trickle using subprocess + +EX: We have a config file contains:: + + [default] + action = backup + storage = ssh + ssh_host = 127.0.0.1 + ssh_username = saad + ssh_key = /home/saad/.ssh/saad + container = /home/saad/backups_freezers + backup_name = freezer_jobs + path_to_backup = /etc + upload_limit=2k + download_limit=1k + +and we are going to execute this job as follow:: + + freezer-agent --config /home/user/job1.ini + +this will be translated to:: + + trickle -u 2048 -d 1024 freezer-agent --config /tmp/freezer_job_x21aj29 + +The new config file has the following arguments:: + + [default] + action = backup + storage = ssh + ssh_host = 127.0.0.1 + ssh_username = saad + ssh_key = /home/saad/.ssh/saad + container = /home/saad/backups_freezers + backup_name = freezer_jobs + path_to_backup = /etc + +3- Hybrid using config file and cli options +we will use a mix of both procedures: +- remove limits (cli or config ) +- reproduce the same command again with trickle +EX:: + + $ freezer-agent --config /home/user/job2.ini --upload-limit 1k + +The Freezer logo is released under the licence Attribution 3.0 Unported (CC BY3.0). + diff --git a/freezer/arguments.py b/freezer/arguments.py index 653ee177..4be44f71 100644 --- a/freezer/arguments.py +++ b/freezer/arguments.py @@ -64,7 +64,8 @@ DEFAULT_PARAMS = { 'vssadmin': False, 'shadow': '', 'shadow_path': '', 'windows_volume': '', 'command': None, 'metadata_out': False, 'storage': 'swift', 'ssh_key': '', 'ssh_username': '', 'ssh_host': '', - 'ssh_port': 22, 'compression': 'gzip', 'overwrite': False + 'ssh_port': 22, 'compression': 'gzip', 'overwrite': False, + 'remove_before_date': False } @@ -202,6 +203,13 @@ def backup_arguments(): 'The option --remove-older-then is deprecated ' 'and will be removed soon'), dest='remove_older_than', type=float, default=None) + arg_parser.add_argument( + '--remove-before-date', action='store', + help=("Checks the specified container and removes objects older than" + " the provided datetime in the form 'YYYY-MM-DDThh:mm:ss' i.e." + " '1974-03-25T23:23:23'. Make sure the 'T' is between date and" + " time , it also supports timestamp values"), + dest='remove_before_date', type=float, default=None) arg_parser.add_argument( '--remove-from-date', action='store', help=('Checks the specified container and removes objects older than ' diff --git a/freezer/job.py b/freezer/job.py index 3d6718ea..e6141f36 100644 --- a/freezer/job.py +++ b/freezer/job.py @@ -14,18 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. """ - -import datetime +import logging import sys -import time from freezer import backup from freezer import exec_cmd from freezer import restore from freezer import utils -import logging - class Job: """ @@ -146,16 +142,23 @@ class RestoreJob(Job): class AdminJob(Job): @Job.executemethod def execute(self): - if self.conf.remove_from_date: - timestamp = utils.date_to_timestamp(self.conf.remove_from_date) - else: - timestamp = datetime.datetime.now() - \ - datetime.timedelta(days=self.conf.remove_older_than) - timestamp = int(time.mktime(timestamp.timetuple())) + if self.conf.remove_before_date: - self.storage.remove_older_than(timestamp, - self.conf.hostname_backup_name) - return {} + if utils.is_iso_date(self.conf.remove_before_date): + timestamp = utils.date_to_timestamp( + self.conf.remove_before_date) + elif utils.is_timestamp(self.conf.remove_before_date): + timestamp = self.conf.remove_before_date + else: + raise Exception('Expecting ISO date or valid timestamp.') + + self.storage.remove_before_date(timestamp, + self.conf.hostname_backup_name) + return {} + elif self.conf.remove_older_than: + self.storage.remove_older_than(self.conf.remove_older_than, + self.conf.hostname_backup_name) + return {} class ExecJob(Job): diff --git a/freezer/main.py b/freezer/main.py index f57742b8..4782a222 100644 --- a/freezer/main.py +++ b/freezer/main.py @@ -176,8 +176,16 @@ def run_job(conf, storage): 'info': job.InfoJob, 'admin': job.AdminJob, 'exec': job.ExecJob}[conf.action](conf, storage) + response = freezer_job.execute() + # Inject remove-from-date functionality for a backup and restore jobs + if ((conf.remove_before_date or conf.remove_from_date or + conf.remove_older_than) and conf.action != 'admin'): + # TODO(m3m0): return something here + job.AdminJob(conf, storage).execute() + # TODO(m3m0): delete all backups from the api. + if conf.metadata_out and response: if conf.metadata_out == '-': sys.stdout.write(json.dumps(response)) diff --git a/freezer/storage/base.py b/freezer/storage/base.py index e02d085b..bd1471ae 100644 --- a/freezer/storage/base.py +++ b/freezer/storage/base.py @@ -14,9 +14,11 @@ from freezer import utils +import abc import logging import os import re +import time class Storage(object): @@ -133,7 +135,7 @@ class Storage(object): """ raise NotImplementedError("Should have implemented this") - def remove_older_than(self, remove_older_timestamp, hostname_backup_name): + def remove_before_date(self, remove_older_timestamp, hostname_backup_name): """ Removes backups which are older than the specified timestamp :type remove_older_timestamp: int @@ -141,10 +143,23 @@ class Storage(object): """ backups = self.find_all(hostname_backup_name) backups = [b for b in backups - if b.latest_update.timestamp < remove_older_timestamp] + if b.latest_update.timestamp <= remove_older_timestamp] for b in backups: b.storage.remove_backup(b) + def remove_older_than(self, days, hostname_backup_name): + """Removes backups older than n amount of days. + :param days: int + :param hostname_backup_name: str + :return: + """ + now = time.time() + seconds_old = utils.days_to_seconds(days) + to_delete_timestamp = now - seconds_old + return self.remove_before_date(to_delete_timestamp, + hostname_backup_name) + + @abc.abstractmethod def info(self): raise NotImplementedError("Should have implemented this") @@ -295,6 +310,7 @@ class Backup: @staticmethod def parse_backups(names, storage): """ + :param names: :type names: list[str] - file names of backups. :type storage: freezer.storage.base.Storage @@ -302,6 +318,7 @@ class Backup: :rtype: list[freezer.storage.base.Backup] :return: list of zero level backups """ + # TODO(m3m0): rename this function prefix = 'tar_metadata_' tar_names = set([x[len(prefix):] for x in names if x.startswith(prefix)]) diff --git a/freezer/storage/swift.py b/freezer/storage/swift.py index 42f04018..78bf104b 100644 --- a/freezer/storage/swift.py +++ b/freezer/storage/swift.py @@ -174,11 +174,11 @@ class SwiftStorage(base.Storage): for i in range(backup.latest_update.level, -1, -1): if i in backup.increments: # remove segment - self.remove(self.segments, backup.increments[i]) + self.remove(self.segments, backup.increments[i].__repr__()) # remove tar self.remove(self.container, backup.increments[i].tar()) # remove manifest - self.remove(self.container, backup.increments[i]) + self.remove(self.container, backup.increments[i].__repr__()) def add_stream(self, stream, package_name, headers=None): i = 0 diff --git a/freezer/tests/commons.py b/freezer/tests/commons.py index 508bfc87..613c85c1 100644 --- a/freezer/tests/commons.py +++ b/freezer/tests/commons.py @@ -293,6 +293,7 @@ class BackupOpt1: self.max_level = '0' self.hostname_backup_name = "hostname_backup_name" self.remove_older_than = '0' + self.remove_before_date = '2016-07-12T11:13:40' self.max_segment_size = '0' self.time_stamp = 123456789 self.container = 'test-container' diff --git a/freezer/utils.py b/freezer/utils.py index e8ea9edd..2f3f28b1 100644 --- a/freezer/utils.py +++ b/freezer/utils.py @@ -24,6 +24,7 @@ import re import subprocess import sys import time +import traceback from distutils import spawn as distspawn from functools import wraps @@ -285,9 +286,12 @@ def create_subprocess(cmd): def date_to_timestamp(date): - fmt = '%Y-%m-%dT%H:%M:%S' - opt_backup_date = datetime.datetime.strptime(date, fmt) - return int(time.mktime(opt_backup_date.timetuple())) + try: + fmt = '%Y-%m-%dT%H:%M:%S' + opt_backup_date = datetime.datetime.strptime(date, fmt) + return int(time.mktime(opt_backup_date.timetuple())) + except Exception: + raise Exception('Invalid ISO date format') class Bunch: @@ -525,3 +529,56 @@ class Namespace(dict): @staticmethod def delattr(ns, name): return object.__delattr__(ns, name) + + +def set_max_process_priority(): + """ Set freezer in max priority on the os """ + # children processes inherit niceness from father + try: + logging.warning( + 'Setting freezer execution with high CPU and I/O priority') + PID = os.getpid() + # Set cpu priority + os.nice(-19) + # Set I/O Priority to Real Time class with level 0 + subprocess.call([ + u'{0}'.format(find_executable("ionice")), + u'-c', u'1', u'-n', u'0', u'-t', + u'-p', u'{0}'.format(PID) + ]) + except Exception as priority_error: + logging.warning('Priority: {0}'.format(priority_error)) + + +def abort_subprocess(signum, frame): + try: + logging.warning('Process {} has been aborted by the scheduler'.format( + os.getpid())) + raise Exception('Aborting process') + except Exception: + logging.error(traceback.print_exc()) + finally: + sys.exit(33) + + +def days_to_seconds(n): + """ + 86400 seconds in a day + :param n: number of days + :return: int + """ + return n * 86400 + + +def is_iso_date(date): + iso_f = re.compile('^(\d{4})-0?(\d+)-0?(\d+)[T ]0?(\d+):0?(\d+):0?(\d+)$') + return re.match(iso_f, date) + + +def is_timestamp(ts): + try: + ts = int(ts) + return True + except ValueError: + raise Exception('Invalid timestamp') + diff --git a/tests/unit/storages/test_base.py b/tests/unit/storages/test_base.py index 425f1fa2..6edaa83f 100644 --- a/tests/unit/storages/test_base.py +++ b/tests/unit/storages/test_base.py @@ -180,10 +180,10 @@ class TestBackup(unittest.TestCase): base.Backup(t, "host_backup", 5000), ] t.remove_backup = mock.Mock() - t.remove_older_than(3000, "host_backup") + t.remove_before_date(3000, "host_backup") t.remove_backup.assert_any_call(r1) t.remove_backup.assert_any_call(r2) - assert t.remove_backup.call_count == 2 + assert t.remove_backup.call_count == 3 def test_create_backup(self): t = base.Storage(None, skip_prepare=True)