openstack
/
freezer
Code Issues Proposed changes

Merge "Fix Backup removal is not working" into stable/liberty

This commit is contained in:
Jenkins 2016-10-11 16:37:25 +00:00 committed by Gerrit Code Review
parent d9c0137867 b5c764032d
commit a0df62b5a3
9 changed files with 213 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
README.rst
View File

@ -348,6 +348,7 @@ Freezer can use:
--storage ssh --ssh-username ubuntu --ssh-key ~/.ssh/id_rsa
--ssh-host 8.8.8.8
**Note** ssh keys with passphrase are not supported at the moment.
Restore
-------
@ -400,9 +401,9 @@ List remote objects in container::
$ sudo freezerc --action info --container freezer_testcontainer -l
Remove backups older then 1 day::
Remove backups older than a date::
$ freezerc --action admin --container freezer_dev-test --remove-older-then 1 --backup-name dev-test-01
$ freezer-agent --action admin --container freezer_dev-test --remove-before-date 2016-07-11T00:00:00 --backup-name dev-test-01
Cinder restore currently creates a volume with the contents of the saved one, but
@ -444,8 +445,9 @@ Freezer architectural components are the following:
Freezer uses GNU Tar under the hood to execute incremental backup and
restore. When a key is provided, it uses OpenSSL to encrypt data.
(AES-256-CFB)
=============
Freezer architecture is composed by the following components:
Freezer components.
-------------------
+-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+
| Component | Description |
@ -987,3 +989,92 @@ optional arguments:
Remote username for ssh storage only
--ssh-host SSH_HOST Remote host for ssh storage only
--ssh-port SSH_PORT Remote port for ssh storage only (default 22)
=======
Scheduler Options
-----------------
To get an updated sample of freezer-scheduler configuration you the following command::
oslo-config-generator --config-file config-generator/scheduler.conf
you will find the update sample file in etc/scheduler.conf.sample
Agent Options
-------------
To list options available in freezer-agent use the following command::
oslo-config-generator --namespace freezer --namespace oslo.log
this will print all options to the screen you direct the output to a file if you want::
oslo-config-generator --namespace freezer --namespace oslo.log --output-file etc/agent.conf.sample
Bandwidth limitation (Trickle)
------------------------------
Trickle for bandwidth limiting ( How it works ):
We have 3 cases to handle
1- User used --upload-limit or --download-limit from the cli
We need to remove these limits from the cli arguments and then run trickle
using subprocess
EX::
# freezer-agent --action backup -F /etc/ -C freezer --upload-limit = 1k
this will be translated to::
# trickle -u 1024 -d -1 freezer-agent --action backup -F /etc/ -C freezer
2- User used config files to execute an action
We need to create a new config file without the limits So we will get the all
the arguments provided and remove limits then run trickle using subprocess
EX: We have a config file contains::
[default]
action = backup
storage = ssh
ssh_host = 127.0.0.1
ssh_username = saad
ssh_key = /home/saad/.ssh/saad
container = /home/saad/backups_freezers
backup_name = freezer_jobs
path_to_backup = /etc
upload_limit=2k
download_limit=1k
and we are going to execute this job as follow::
freezer-agent --config /home/user/job1.ini
this will be translated to::
trickle -u 2048 -d 1024 freezer-agent --config /tmp/freezer_job_x21aj29
The new config file has the following arguments::
[default]
action = backup
storage = ssh
ssh_host = 127.0.0.1
ssh_username = saad
ssh_key = /home/saad/.ssh/saad
container = /home/saad/backups_freezers
backup_name = freezer_jobs
path_to_backup = /etc
3- Hybrid using config file and cli options
we will use a mix of both procedures:
- remove limits (cli or config )
- reproduce the same command again with trickle
EX::
$ freezer-agent --config /home/user/job2.ini --upload-limit 1k
The Freezer logo is released under the licence Attribution 3.0 Unported (CC BY3.0).

10
freezer/arguments.py
View File

@ -64,7 +64,8 @@ DEFAULT_PARAMS = {
'vssadmin': False, 'shadow': '', 'shadow_path': '',
'windows_volume': '', 'command': None, 'metadata_out': False,
'storage': 'swift', 'ssh_key': '', 'ssh_username': '', 'ssh_host': '',
'ssh_port': 22, 'compression': 'gzip', 'overwrite': False
'ssh_port': 22, 'compression': 'gzip', 'overwrite': False,
'remove_before_date': False
}
@ -202,6 +203,13 @@ def backup_arguments():
'The option --remove-older-then is deprecated '
'and will be removed soon'),
dest='remove_older_than', type=float, default=None)
arg_parser.add_argument(
'--remove-before-date', action='store',
help=("Checks the specified container and removes objects older than"
" the provided datetime in the form 'YYYY-MM-DDThh:mm:ss' i.e."
" '1974-03-25T23:23:23'. Make sure the 'T' is between date and"
" time , it also supports timestamp values"),
dest='remove_before_date', type=float, default=None)
arg_parser.add_argument(
'--remove-from-date', action='store',
help=('Checks the specified container and removes objects older than '

31
freezer/job.py
View File

@ -14,18 +14,14 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
import datetime
import logging
import sys
import time
from freezer import backup
from freezer import exec_cmd
from freezer import restore
from freezer import utils
import logging
class Job:
"""
@ -146,16 +142,23 @@ class RestoreJob(Job):
class AdminJob(Job):
@Job.executemethod
def execute(self):
if self.conf.remove_from_date:
timestamp = utils.date_to_timestamp(self.conf.remove_from_date)
else:
timestamp = datetime.datetime.now() - \
datetime.timedelta(days=self.conf.remove_older_than)
timestamp = int(time.mktime(timestamp.timetuple()))
if self.conf.remove_before_date:
self.storage.remove_older_than(timestamp,
self.conf.hostname_backup_name)
return {}
if utils.is_iso_date(self.conf.remove_before_date):
timestamp = utils.date_to_timestamp(
self.conf.remove_before_date)
elif utils.is_timestamp(self.conf.remove_before_date):
timestamp = self.conf.remove_before_date
else:
raise Exception('Expecting ISO date or valid timestamp.')
self.storage.remove_before_date(timestamp,
self.conf.hostname_backup_name)
return {}
elif self.conf.remove_older_than:
self.storage.remove_older_than(self.conf.remove_older_than,
self.conf.hostname_backup_name)
return {}
class ExecJob(Job):

8
freezer/main.py
View File

@ -176,8 +176,16 @@ def run_job(conf, storage):
'info': job.InfoJob,
'admin': job.AdminJob,
'exec': job.ExecJob}[conf.action](conf, storage)
response = freezer_job.execute()
# Inject remove-from-date functionality for a backup and restore jobs
if ((conf.remove_before_date or conf.remove_from_date or
conf.remove_older_than) and conf.action != 'admin'):
# TODO(m3m0): return something here
job.AdminJob(conf, storage).execute()
# TODO(m3m0): delete all backups from the api.
if conf.metadata_out and response:
if conf.metadata_out == '-':
sys.stdout.write(json.dumps(response))

21
freezer/storage/base.py
View File

@ -14,9 +14,11 @@
from freezer import utils
import abc
import logging
import os
import re
import time
class Storage(object):
@ -133,7 +135,7 @@ class Storage(object):
"""
raise NotImplementedError("Should have implemented this")
def remove_older_than(self, remove_older_timestamp, hostname_backup_name):
def remove_before_date(self, remove_older_timestamp, hostname_backup_name):
"""
Removes backups which are older than the specified timestamp
:type remove_older_timestamp: int
@ -141,10 +143,23 @@ class Storage(object):
"""
backups = self.find_all(hostname_backup_name)
backups = [b for b in backups
if b.latest_update.timestamp < remove_older_timestamp]
if b.latest_update.timestamp <= remove_older_timestamp]
for b in backups:
b.storage.remove_backup(b)
def remove_older_than(self, days, hostname_backup_name):
"""Removes backups older than n amount of days.
:param days: int
:param hostname_backup_name: str
:return:
"""
now = time.time()
seconds_old = utils.days_to_seconds(days)
to_delete_timestamp = now - seconds_old
return self.remove_before_date(to_delete_timestamp,
hostname_backup_name)
@abc.abstractmethod
def info(self):
raise NotImplementedError("Should have implemented this")
@ -295,6 +310,7 @@ class Backup:
@staticmethod
def parse_backups(names, storage):
"""
:param names:
:type names: list[str] - file names of backups.
:type storage: freezer.storage.base.Storage
@ -302,6 +318,7 @@ class Backup:
:rtype: list[freezer.storage.base.Backup]
:return: list of zero level backups
"""
# TODO(m3m0): rename this function
prefix = 'tar_metadata_'
tar_names = set([x[len(prefix):]
for x in names if x.startswith(prefix)])

4
freezer/storage/swift.py
View File

@ -174,11 +174,11 @@ class SwiftStorage(base.Storage):
for i in range(backup.latest_update.level, -1, -1):
if i in backup.increments:
# remove segment
self.remove(self.segments, backup.increments[i])
self.remove(self.segments, backup.increments[i].__repr__())
# remove tar
self.remove(self.container, backup.increments[i].tar())
# remove manifest
self.remove(self.container, backup.increments[i])
self.remove(self.container, backup.increments[i].__repr__())
def add_stream(self, stream, package_name, headers=None):
i = 0

1
freezer/tests/commons.py
View File

@ -293,6 +293,7 @@ class BackupOpt1:
self.max_level = '0'
self.hostname_backup_name = "hostname_backup_name"
self.remove_older_than = '0'
self.remove_before_date = '2016-07-12T11:13:40'
self.max_segment_size = '0'
self.time_stamp = 123456789
self.container = 'test-container'

63
freezer/utils.py
View File

@ -24,6 +24,7 @@ import re
import subprocess
import sys
import time
import traceback
from distutils import spawn as distspawn
from functools import wraps
@ -285,9 +286,12 @@ def create_subprocess(cmd):
def date_to_timestamp(date):
fmt = '%Y-%m-%dT%H:%M:%S'
opt_backup_date = datetime.datetime.strptime(date, fmt)
return int(time.mktime(opt_backup_date.timetuple()))
try:
fmt = '%Y-%m-%dT%H:%M:%S'
opt_backup_date = datetime.datetime.strptime(date, fmt)
return int(time.mktime(opt_backup_date.timetuple()))
except Exception:
raise Exception('Invalid ISO date format')
class Bunch:
@ -525,3 +529,56 @@ class Namespace(dict):
@staticmethod
def delattr(ns, name):
return object.__delattr__(ns, name)
def set_max_process_priority():
""" Set freezer in max priority on the os """
# children processes inherit niceness from father
try:
logging.warning(
'Setting freezer execution with high CPU and I/O priority')
PID = os.getpid()
# Set cpu priority
os.nice(-19)
# Set I/O Priority to Real Time class with level 0
subprocess.call([
u'{0}'.format(find_executable("ionice")),
u'-c', u'1', u'-n', u'0', u'-t',
u'-p', u'{0}'.format(PID)
])
except Exception as priority_error:
logging.warning('Priority: {0}'.format(priority_error))
def abort_subprocess(signum, frame):
try:
logging.warning('Process {} has been aborted by the scheduler'.format(
os.getpid()))
raise Exception('Aborting process')
except Exception:
logging.error(traceback.print_exc())
finally:
sys.exit(33)
def days_to_seconds(n):
"""
86400 seconds in a day
:param n: number of days
:return: int
"""
return n * 86400
def is_iso_date(date):
iso_f = re.compile('^(\d{4})-0?(\d+)-0?(\d+)[T ]0?(\d+):0?(\d+):0?(\d+)$')
return re.match(iso_f, date)
def is_timestamp(ts):
try:
ts = int(ts)
return True
except ValueError:
raise Exception('Invalid timestamp')

4
tests/unit/storages/test_base.py
View File

@ -180,10 +180,10 @@ class TestBackup(unittest.TestCase):
base.Backup(t, "host_backup", 5000),
]
t.remove_backup = mock.Mock()
t.remove_older_than(3000, "host_backup")
t.remove_before_date(3000, "host_backup")
t.remove_backup.assert_any_call(r1)
t.remove_backup.assert_any_call(r2)
assert t.remove_backup.call_count == 2
assert t.remove_backup.call_count == 3
def test_create_backup(self):
t = base.Storage(None, skip_prepare=True)