openstack
/
freezer
Code Issues Proposed changes

Fix Devstack gate job and Documentation issues 

Change-Id: If9c761643f4924bf683c65f23152dba8df388fe9
This commit is contained in:
Saad Zaher 2017-12-02 23:09:58 +00:00
parent 27d5c7bd67
commit 0c8406cce4
7 changed files with 28 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
devstack/gate_hook.sh
View File

@ -27,6 +27,8 @@ echo "Start Gate Hook"
# Swift is needed for some of the integration tests
export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_service s-proxy s-object s-container s-account"
export DEVSTACK_GATE_TEMPEST_REGEX="freezer_tempest_plugin"
$BASE/new/devstack-gate/devstack-vm-gate.sh
echo "End Gate Hook"

4
doc/source/_static/freezer.conf.sample
View File

@ -91,6 +91,10 @@
# log_config_append is set. (string value)
#syslog_log_facility = LOG_USER
# Use JSON formatting for logging. This option is ignored if log_config_append
# is set. (boolean value)
#use_json = false
# Log output to standard error. This option is ignored if log_config_append is
# set. (boolean value)
#use_stderr = false

13
doc/source/conf.py
View File

@ -39,10 +39,10 @@ import sys
# They can be extensions coming with Sphinx (named 'sphinx.ext.*')
# or your custom ones.
extensions = ['openstackdocstheme',
extensions = ['sphinx.ext.autodoc',
'openstackdocstheme',
'oslo_config.sphinxconfiggen',
'oslo_config.sphinxext',
'sphinx.ext.autodoc',
'oslo_config.sphinxext'
]
config_generator_config_file = (
@ -144,6 +144,13 @@ man_pages = [
# -- Options for HTML output --------------------------------------------------
# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = True
# The theme to use for HTML and HTML Help pages. Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
# html_theme_path = ["."]

304
doc/source/user/freezer-agent.rst
View File

@ -573,310 +573,6 @@ The scheduler can be started on the target node with:
The scheduler could have already been started. As soon as the freezer-scheduler
contacts the API, it fetches the job and schedules it.
Freezer Agent Options
=====================
Available Options
-----------------
.. code:: bash
usage: freezer-agent [-h] [--action ACTION] [--always-level ALWAYS_LEVEL]
[--backup-name BACKUP_NAME]
[--cinder-vol-id CINDER_VOL_ID]
[--cindernative-vol-id CINDERNATIVE_VOL_ID]
[--command COMMAND] [--compression COMPRESSION]
[--config CONFIG] [--config-dir DIR] [--config-file PATH]
[--container CONTAINER] [--debug]
[--dereference-symlink DEREFERENCE_SYMLINK]
[--download-limit DOWNLOAD_LIMIT] [--dry-run]
[--encrypt-pass-file ENCRYPT_PASS_FILE]
[--exclude EXCLUDE] [--hostname HOSTNAME] [--insecure]
[--log-config-append PATH]
[--log-date-format DATE_FORMAT] [--log-dir LOG_DIR]
[--log-file PATH] [--log-format FORMAT]
[--lvm-auto-snap LVM_AUTO_SNAP]
[--lvm-dirmount LVM_DIRMOUNT]
[--lvm-snap-perm LVM_SNAPPERM]
[--lvm-snapname LVM_SNAPNAME]
[--lvm-snapsize LVM_SNAPSIZE] [--lvm-srcvol LVM_SRCVOL]
[--lvm-volgroup LVM_VOLGROUP] [--max-level MAX_LEVEL]
[--max-priority MAX_PRIORITY]
[--max-segment-size MAX_SEGMENT_SIZE]
[--metadata-out METADATA_OUT] [--mode MODE]
[--mysql-conf MYSQL_CONF]
[--no-incremental NO_INCREMENTAL] [--nodebug]
[--nodry-run] [--noinsecure] [--nooverwrite] [--noquiet]
[--nouse-syslog] [--nouse-syslog-rfc-format]
[--nova-inst-id NOVA_INST_ID] [--noverbose]
[--nowatch-log-file]
[--os-identity-api-version OS_IDENTITY_API_VERSION]
[--overwrite] [--path-to-backup PATH_TO_BACKUP]
[--proxy PROXY] [--quiet]
[--remove-from-date REMOVE_FROM_DATE]
[--remove-older-than REMOVE_OLDER_THAN]
[--restart-always-level RESTART_ALWAYS_LEVEL]
[--restore-abs-path RESTORE_ABS_PATH]
[--restore-from-date RESTORE_FROM_DATE]
[--snapshot SNAPSHOT] [--sql-server-conf SQL_SERVER_CONF]
[--ssh-host SSH_HOST] [--ssh-key SSH_KEY]
[--ssh-port SSH_PORT] [--ssh-username SSH_USERNAME]
[--storage STORAGE]
[--syslog-log-facility SYSLOG_LOG_FACILITY]
[--upload-limit UPLOAD_LIMIT] [--use-syslog]
[--use-syslog-rfc-format] [--verbose] [--version]
[--watch-log-file]
Optional Arguments
------------------
-h, --help show this help message and exit
--action ACTION Set the action to be taken. backup and restore are
self explanatory, info is used to retrieve info from
the storage media, exec is used to execute a script,
while admin is used to delete old backups and other
admin actions. Default backup.
--always-level ALWAYS_LEVEL
Set backup maximum level used with tar to implement
incremental backup. If a level 3 is specified, the
backup will be executed from level 0 to level 3 and to
that point always a backup level 3 will be executed.
It will not restart from level 0. This option has
precedence over --max-backup-level. Default False
(Disabled)
--backup-name BACKUP_NAME, -N BACKUP_NAME
The backup name you want to use to identify your
backup on Swift
--cinder-vol-id CINDER_VOL_ID
Id of cinder volume for backup
--cindernative-vol-id CINDERNATIVE_VOL_ID
Id of cinder volume for native backup
--command COMMAND Command executed by exec action
--compression COMPRESSION
compression algorithm to use. gzip is default
algorithm
--config CONFIG Config file abs path. Option arguments are provided
from config file. When config file is used any option
from command line provided take precedence.
--config-dir DIR Path to a config directory to pull *.conf files from.
This file set is sorted, so as to provide a
predictable parse order if individual options are
over-ridden. The set is parsed after the file(s)
specified via previous --config-file, arguments hence
over-ridden options in the directory take precedence.
--config-file PATH Path to a config file to use. Multiple config files
can be specified, with values in later files taking
precedence. Defaults to None.
--container CONTAINER, -C CONTAINER
The Swift container (or path to local storage) used to
upload files to
--debug, -d If set to true, the logging level will be set to DEBUG
instead of the default INFO level.
--dereference-symlink DEREFERENCE_SYMLINK
Follow hard and soft links and archive and dump the
files they refer to. Default False.
--download-limit DOWNLOAD_LIMIT
Download bandwidth limit in Bytes per sec. Can be
invoked with dimensions (10K, 120M, 10G).
--dry-run Do everything except writing or removing objects
--encrypt-pass-file ENCRYPT_PASS_FILE
Passing a private key to this option, allow you to
encrypt the files before to be uploaded in Swift.
Default do not encrypt.
--exclude EXCLUDE Exclude files,given as a PATTERN.Ex: --exclude '*.log'
will exclude any file with name ending with .log.
Default no exclude
--hostname HOSTNAME, --restore_from_host HOSTNAME
Set hostname to execute actions. If you are executing
freezer from one host but you want to delete objects
belonging to another host then you can set this option
that hostname and execute appropriate actions. Default
current node hostname.
--insecure Allow to access swift servers without checking SSL
certs.
--log-config-append PATH, --log_config PATH
The name of a logging configuration file. This file is
appended to any existing logging configuration files.
For details about logging configuration files, see the
Python logging module documentation. Note that when
logging configuration files are used all logging
configuration is defined in the configuration file and
other logging configuration options are ignored.
--log-date-format DATE_FORMAT
Defines the format string for %(asctime)s in log
records. Default: None . This option is ignored if
log_config_append is set.
--log-dir LOG_DIR, --logdir LOG_DIR
(Optional) The base directory used for relative
log_file paths. This option is ignored if
log_config_append is set.
--log-file PATH, --logfile PATH
(Optional) Name of log file to send logging output to.
If no default is set, logging will go to stderr as
defined by use_stderr. This option is ignored if
log_config_append is set.
--log-format FORMAT DEPRECATED. A logging.Formatter log message format
string which may use any of the available
logging.LogRecord attributes. This option is
deprecated. Please use logging_context_format_string
and logging_default_format_string instead. This option
is ignored if log_config_append is set.
--lvm-auto-snap LVM_AUTO_SNAP
Automatically guess the volume group and volume name
for given PATH.
--lvm-dirmount LVM_DIRMOUNT
Set the directory you want to mount the lvm snapshot
to. If not provided, a unique name will be generated
with thebasename /var/lib/freezer
--lvm-snap-perm LVM_SNAPPERM
Set the lvm snapshot permission to use. If the
permission is set to ro The snapshot will be immutable
- read only -. If the permission is set to rw it will
be mutable
--lvm-snapname LVM_SNAPNAME
Set the name of the snapshot that will be created. If
not provided, a unique name will be generated.
--lvm-snapsize LVM_SNAPSIZE
Set the lvm snapshot size when creating a new
snapshot. Please add G for Gigabytes or M for
Megabytes, i.e. 500M or 8G. It is also possible to use
percentages as with the -l option of lvm, i.e. 80%FREE
Default 1G.
--lvm-srcvol LVM_SRCVOL
Set the lvm volume you want to take a snaphost from.
Default no volume
--lvm-volgroup LVM_VOLGROUP
Specify the volume group of your logical volume. This
is important to mount your snapshot volume. Default
not set
--max-level MAX_LEVEL
Set the backup level used with tar to implement
incremental backup. If a level 1 is specified but no
level 0 is already available, a level 0 will be done
and subsequently backs to level 1. Default 0 (No
Incremental)
--max-priority MAX_PRIORITY
Set the cpu process to the highest priority (i.e. -20
on Linux) and real-time for I/O. The process priority
will be set only if nice and ionice are installed
Default disabled. Use with caution.
--max-segment-size MAX_SEGMENT_SIZE, -M MAX_SEGMENT_SIZE
Set the maximum file chunk size in bytes to upload to
swift Default 33554432 bytes (32MB)
--metadata-out METADATA_OUT
Set the filename to which write the metadata regarding
the backup metrics. Use '-' to output to standard
output.
--mode MODE, -m MODE Set the technology to back from. Options are, fs
(filesystem),mongo (MongoDB), mysql (MySQL), sqlserver
(SQL Server) Default set to fs
--mysql-conf MYSQL_CONF
Set the MySQL configuration file where freezer
retrieve important information as db_name, user,
password, host, port. Following is an example of
config file: # backup_mysql_confhost = <db-host>user =
<mysqluser>password = <mysqlpass>port = <db-port>
--no-incremental NO_INCREMENTAL
Disable incremental feature. By default freezer build
the meta data even for level 0 backup. By setting this
option incremental meta data is not created at all.
Default disabled
--nodebug The inverse of --debug
--nodry-run The inverse of --dry-run
--noinsecure The inverse of --insecure
--nooverwrite The inverse of --overwrite
--noquiet The inverse of --quiet
--nouse-syslog The inverse of --use-syslog
--nouse-syslog-rfc-format
The inverse of --use-syslog-rfc-format
--nova-inst-id NOVA_INST_ID
Id of nova instance for backup
--noverbose The inverse of --verbose
--nowatch-log-file The inverse of --watch-log-file
--os-identity-api-version OS_IDENTITY_API_VERSION, --os_auth_ver OS_IDENTITY_API_VERSION
OpenStack identity api version, can be 1, 2, 2.0 or 3
--overwrite With overwrite removes files from restore path before
restore.
--path-to-backup PATH_TO_BACKUP, -F PATH_TO_BACKUP
The file or directory you want to back up to Swift
--proxy PROXY Enforce proxy that alters system HTTP_PROXY and
HTTPS_PROXY, use '' to eliminate all system proxies
--quiet, -q Suppress error messages
--remove-from-date REMOVE_FROM_DATE
Checks the specified container and removes objects
older than the provided datetime in the form 'YYYY-MM-
DDThh:mm:ss' i.e. '1974-03-25T23:23:23'. Make sure the
'T' is between date and time
--remove-older-than REMOVE_OLDER_THAN, -R REMOVE_OLDER_THAN
Checks in the specified container for object older
than the specified days. If i.e. 30 is specified, it
will remove the remote object older than 30 days.
Default False (Disabled) The option --remove-older-
then is deprecated and will be removed soon
--restart-always-level RESTART_ALWAYS_LEVEL
Restart the backup from level 0 after n days. Valid
only if --always-level option if set. If --always-
level is used together with --remove-older-than, there
might be the chance where the initial level 0 will be
removed. Default False (Disabled)
--restore-abs-path RESTORE_ABS_PATH
Set the absolute path where you want your data
restored. Default False.
--restore-from-date RESTORE_FROM_DATE
Set the date of the backup from which you want to
restore.This will select the most recent backup
previous to the specified date (included). Example: if
the last backup was created at '2016-03-22T14:29:01'
and restore-from-date is set to '2016-03-22T14:29:01',
the backup will be restored successfully. The same for
any date after that, even if the provided date is in
the future. However if restore-from-date is set to
'2016-03-22T14:29:00' or before, that backup will not
be found. Please provide datetime in format 'YYYY-MM-
DDThh:mm:ss' i.e. '1979-10-03T23:23:23'. Make sure the
'T' is between date and time Default None.
--snapshot SNAPSHOT, -s SNAPSHOT
Create a snapshot of the fs containing the resource to
backup. When used, the lvm parameters will be guessed
and/or the default values will be used, on windows it
will invoke vssadmin
--sql-server-conf SQL_SERVER_CONF
Set the SQL Server configuration file where freezer
retrieve the sql server instance. Following is an
example of config file: instance = <db-instance>
--ssh-host SSH_HOST Remote host for ssh storage only
--ssh-key SSH_KEY Path to ssh-key for ssh storage only
--ssh-port SSH_PORT Remote port for ssh storage only (default 22)
--ssh-username SSH_USERNAME
Remote username for ssh storage only
--storage STORAGE Storage for backups. Can be Swift or Local now. Swift
is default storage now. Local stores backups on the
same defined path and swift will store files in
container.
--syslog-log-facility SYSLOG_LOG_FACILITY
Syslog facility to receive log lines. This option is
ignored if log_config_append is set.
--upload-limit UPLOAD_LIMIT
Upload bandwidth limit in Bytes per sec. Can be
invoked with dimensions (10K, 120M, 10G).
--use-syslog Use syslog for logging. Existing syslog format is
DEPRECATED and will be changed later to honor RFC5424.
This option is ignored if log_config_append is set.
--use-syslog-rfc-format
Enables or disables syslog rfc5424 format for logging.
If enabled, prefixes the MSG part of the syslog
message with APP-NAME (RFC5424). This option is
ignored if log_config_append is set.
--verbose, -v If set to false, the logging level will be set to
WARNING instead of the default INFO level.
--version show program's version number and exit
--watch-log-file Uses logging handler designed to watch file system.
When log file is moved or removed this handler will
open a new log file with specified path
instantaneously. It makes sense only if log_file
option is specified and Linux platform is used. This
option is ignored if log_config_append is set.
MISC
====

4
freezer/engine/osbrick/volume_actions.py
View File

@ -11,7 +11,7 @@
# under the License.
from os_brick import exception
from os_brick.initiator import connector
from os_brick import initiator
class VolumeAction(object):
@ -44,7 +44,7 @@ class InitializeConnection(VolumeAction):
class VerifyProtocol(VolumeAction):
# NOTE(e0ne): Only iSCSI and RBD based drivers are supported. NFS doesn't
# work. Drivers with other protocols are not tested yet.
SUPPORTED_PROCOTOLS = [connector.ISCSI, connector.RBD]
SUPPORTED_PROCOTOLS = [initiator.ISCSI, initiator.RBD]
def verify(self, protocol):
protocol = protocol.upper()

5
freezer/tests/integration/test_agent.py
View File

@ -41,9 +41,10 @@ class TestBackupFSLocalstorage(common.TestFS):
self.assertTreesMatchNot()
def test_backup_single_level(self):
"""
"""How it works?
- use the default source and destination trees in /tmp
(see common.TestFS)
(see common.TestFS)
- use temporary directory for backup storage
- add some random data
- check that trees don't match anymore

12
freezer/tests/integration/test_rsync_backup.py
View File

@ -22,9 +22,10 @@ from freezer.tests.integration import common
class TestBackupFSLocalRsync(common.TestFS):
def test_backup_single_level(self):
"""
"""How it works?
- use the default source and destination trees in /tmp
(see common.TestFS)
(see common.TestFS)
- use temporary directory for backup storage
- add some random data
- check that trees don't match anymore
@ -64,9 +65,10 @@ class TestBackupFSLocalRsync(common.TestFS):
self.assertTreesMatch()
def test_backup_multiple_level(self):
"""
"""How it works?
- use the default source and destination trees in /tmp
(see common.TestFS)
(see common.TestFS)
- use temporary directory for backup storage
- add some random data
- check that trees don't match anymore
@ -119,7 +121,7 @@ class TestBackupFSLocalRsync(common.TestFS):
def test_backup_single_file(self):
"""
- use the default source and destination trees in /tmp
(see common.TestFS)
(see common.TestFS)
- use temporary directory for backup storage
- add one file with random data
- check that trees don't match anymore