openstack
/
freezer-api
Code Issues Proposed changes

update type deletion policy in db-init 

Types with not-matching mappings are not deleted by default.
When elasticsearch fails to merge the new mapping with the
existing one an error is returned.

The type is deleted only when explicitly requested, either
interactively or providing the following command-line parameter:

    --erase

Adds the possibility to select the specific mapping to upload
with the following command-line parameter:

 --mapping <specific-mapping>

Change-Id: If18fdba770790d8af03475d45da28c2e40fb7da6
This commit is contained in:
Fabrizio Vanni 2015-09-02 18:27:49 +01:00 committed by Pierre-Arthur MATHIEU
parent 7eb2c5b666
commit b5ac449ec2
4 changed files with 265 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
README.rst
View File

@ -10,13 +10,7 @@ Freezer API
-----------------------------
::
# pip install keystonemiddleware falcon
Elasticsearch support
::
# pip install elasticsearch
# pip install -r requirements.txt
1.2 Install freezer_api
-----------------------
@ -25,25 +19,51 @@ Elasticsearch support
# git clone https://github.com/stackforge/freezer-api.git
# cd freezer-api && sudo python setup.py install
this will install into /usr/local
1.3 edit config file
--------------------
::
# sudp cp etc/freezer-api.conf /etc/freezer-api.conf
# sudo cp etc/freezer-api.conf /etc/freezer-api.conf
# sudo vi /etc/freezer-api.conf
1.4 run simple instance
1.4 setup/configure the db
--------------------------
The currently supported db is Elasticsearch. In case you are using a dedicated instance
of the server, you'll need to start it. Depending on the OS flavor it might be a:
::
# service elasticsearch start
or, on systemd::
# systemctl start elasticsearch
Elasticsearch needs to know what type of data each document's field contains.
This information is contained in the "mapping", or schema definition.
Elasticsearch will use dynamic mapping to try to guess the field type from
the basic datatypes available in JSON, but some field's properties have to be
explicitly declared to tune the indexing engine.
To do that, use the freezer-db-init command:
::
# freezer-db-init [db-host]
The url of the db-host is optional and can be automatically guessed from
/etc/freezer-api.conf
To get information about optional additional parameters:
::
freezer-db-init -h
1.5 run simple instance
-----------------------
::
# freezer-api
1.5 examples running using uwsgi
1.6 examples running using uwsgi
--------------------------------
::
@ -70,13 +90,13 @@ backups which share the same container,hostname and backupname
===================
::
keystone user-create --name freezer --pass FREEZER_PWD
keystone user-role-add --user freezer --tenant service --role admin
# keystone user-create --name freezer --pass FREEZER_PWD
# keystone user-role-add --user freezer --tenant service --role admin
keystone service-create --name freezer --type backup \
# keystone service-create --name freezer --type backup \
--description "Freezer Backup Service"
keystone endpoint-create \
# keystone endpoint-create \
--service-id $(keystone service-list | awk '/ backup / {print $2}') \
--publicurl http://freezer_api_publicurl:port \
--internalurl http://freezer_api_internalurl:port \

98
freezer_api/cmd/db_init.py
View File

@ -37,16 +37,19 @@ DEFAULT_ES_SERVER_PORT = 9200
DEFAULT_INDEX = 'freezer'
class MergeMappingException(Exception):
pass
class ElastichsearchEngine(object):
def __init__(self, es_url, es_index, test_only, always_yes, verbose):
def __init__(self, es_url, es_index, args):
self.es_url = es_url
self.es_index = es_index
self.test_only = test_only
self.always_yes = always_yes
self.verbose = verbose
self.args = args
self.exit_code = os.EX_OK
def verbose_print(self, message):
if self.verbose:
def verbose_print(self, message, level=1):
if self.args.verbose >= level:
print(message)
def put_mappings(self, mappings):
@ -56,6 +59,7 @@ class ElastichsearchEngine(object):
print '{0}/{1} MATCHES'.format(self.es_index, es_type)
else:
self.askput_mapping(es_type, mapping)
return self.exit_code
def check_index_exists(self):
url = '{0}/{1}'.format(self.es_url, self.es_index)
@ -80,16 +84,47 @@ class ElastichsearchEngine(object):
return mapping == current_mappings.get(es_type, {})
def askput_mapping(self, es_type, mapping):
if self.test_only:
if self.args.test_only:
print '{0}/{1} DOES NOT MATCH'.format(self.es_index, es_type)
self.exit_code = os.EX_DATAERR
return
prompt_message = ('{0}/{1}/{2} needs to be deleted. '
prompt_message = ('{0}/{1}/{2} needs to be updated. '
'Proceed ? (y/n)'
.format(self.es_url,
self.es_index,
es_type))
if not self.proceed(prompt_message, self.args.yes):
return
self.verbose_print('Trying to upload mappings ...')
try:
self.put_mapping(es_type, mapping)
except MergeMappingException as e:
self.verbose_print('Unable to merge mappings.')
self.verbose_print(e, 2)
else:
print "Mappings updated"
return
if self.args.yes and not self.args.erase:
# explicit consent to update without explicit consent to erase:
# do not erase type and return error code
self.exit_code = os.EX_DATAERR
print ('{0}/{1} DOES NOT MATCH. '
'Need explicit consent to erase types'
.format(self.es_index, es_type))
return
prompt_message = ('Type {0}/{1}/{2} needs to be deleted. '
'Proceed (y/n) ? '.format(self.es_url,
self.es_index,
es_type))
if self.always_yes or self.proceed(prompt_message):
self.delete_type(es_type)
self.put_mapping(es_type, mapping)
if not self.proceed(prompt_message, self.args.erase):
return
self.verbose_print('Deleting type {0}'.format(es_type))
self.delete_type(es_type)
self.verbose_print('Uploading mappings ...')
self.put_mapping(es_type, mapping)
def delete_type(self, es_type):
url = '{0}/{1}/{2}'.format(self.es_url, self.es_index, es_type)
@ -110,11 +145,11 @@ class ElastichsearchEngine(object):
if r.status_code == requests.codes.OK:
print "Type {0} mapping created".format(url)
else:
raise Exception('Type mapping creation error {0}: '
'{1}'.format(r.status_code, r.text))
raise MergeMappingException('Type mapping creation error {0}: '
'{1}'.format(r.status_code, r.text))
def proceed(self, message):
if self.always_yes:
def proceed(self, message, assume_yes=False):
if assume_yes:
return True
while True:
selection = raw_input(message)
@ -124,7 +159,7 @@ class ElastichsearchEngine(object):
return False
def get_args():
def get_args(mapping_choices):
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
'host', action='store', default='', nargs='?',
@ -134,16 +169,27 @@ def get_args():
help=('The DB server port '
'(default: {0})'.format(DEFAULT_ES_SERVER_PORT)),
dest='port', default=0)
arg_parser.add_argument(
'-m', '--mapping', action='store',
help=('Specific mapping to upload. Valid choices: {0}'
.format(','.join(mapping_choices))),
choices=mapping_choices,
dest='select_mapping', default='')
arg_parser.add_argument(
'-i', '--index', action='store',
help='The DB index (default "{0}")'.format(DEFAULT_INDEX),
dest='index')
arg_parser.add_argument(
'-y', '--yes', action='store_true',
help="Automatic confirmation to index removal",
help="Automatic confirmation to mapping update",
dest='yes', default=False)
arg_parser.add_argument(
'-v', '--verbose', action='store_true',
'-e', '--erase', action='store_true',
help=("Enable index deletion in case mapping update "
"fails due to incompatible changes"),
dest='erase', default=False)
arg_parser.add_argument(
'-v', '--verbose', action='count',
help="Verbose",
dest='verbose', default=False)
arg_parser.add_argument(
@ -241,29 +287,29 @@ def get_db_params(args):
def main():
args = get_args()
mappings = db_mappings.get_mappings()
args = get_args(mapping_choices=mappings.keys())
elasticsearch_url, elasticsearch_index = get_db_params(args)
es_manager = ElastichsearchEngine(es_url=elasticsearch_url,
es_index=elasticsearch_index,
test_only=args.test_only,
always_yes=args.yes,
verbose=args.verbose)
args=args)
if args.verbose:
print " db url: {0}".format(elasticsearch_url)
print "db index: {0}".format(elasticsearch_index)
mappings = db_mappings.get_mappings()
if args.select_mapping:
mappings = {args.select_mapping: mappings[args.select_mapping]}
try:
es_manager.put_mappings(mappings)
exit_code = es_manager.put_mappings(mappings)
except Exception as e:
print "ERROR {0}".format(e)
return os.EX_DATAERR
return os.EX_OK
return exit_code
if __name__ == '__main__':
sys.exit(main())

206
freezer_api/common/db_mappings.py
View File

@ -21,121 +21,161 @@ Hudson (tjh@cryptsoft.com).
clients_mapping = {
u'properties': {
u'client': {
u'properties': {
u'client_id': {
u'index': u'not_analyzed',
u'type': u'string',
"properties": {
"client": {
"properties": {
"client_id": {
"index": "not_analyzed",
"type": "string",
},
u'config_id': {
u'index': u'not_analyzed',
u'type': u'string',
"config_id": {
"index": "not_analyzed",
"type": "string",
},
u'description': {
u'type': u'string',
"description": {
"type": "string",
},
u'hostname': {
u'type': u'string',
"hostname": {
"type": "string",
},
},
},
u'user_id': {
u'index': u'not_analyzed',
u'type': u'string',
"user_id": {
"index": "not_analyzed",
"type": "string",
},
"uuid": {
"index": "not_analyzed",
"type": "string"
},
},
}
backups_mapping = {
u'properties': {
u'backup_id': {
u'index': u'not_analyzed',
u'type': u'string',
"properties": {
"backup_id": {
"index": "not_analyzed",
"type": "string",
},
u'backup_metadata': {
u'properties': {
u'backup_name': {
u'index': u'not_analyzed',
u'type': u'string',
"backup_metadata": {
"properties": {
"action": {
"type": "string",
},
u'backup_session': {
u'type': u'long',
"always_level": {
"type": "boolean",
},
u'backup_size_compressed': {
u'type': u'long',
"backup_media": {
"type": "string",
},
u'backup_size_uncompressed': {
u'type': u'long',
"backup_name": {
"index": "not_analyzed",
"type": "string",
},
u'broken_links': {
u'index': u'not_analyzed',
u'type': u'string',
"backup_session": {
"type": "long",
},
u'cli': {
u'type': u'string',
"backup_size_compressed": {
"type": "long",
},
u'client_os': {
u'type': u'string',
"backup_size_uncompressed": {
"type": "long",
},
u'compression_alg': {
u'type': u'string',
"broken_links": {
"index": "not_analyzed",
"type": "string",
},
u'container': {
u'index': u'not_analyzed',
u'type': u'string',
"cli": {
"type": "string",
},
u'encrypted': {
u'type': u'boolean',
"client_os": {
"type": "string",
},
u'excluded_files': {
u'type': u'string',
"client_version": {
"type": "string",
},
u'fs_real_path': {
u'type': u'string',
"compression_alg": {
"type": "string",
},
u'host_name': {
u'index': u'not_analyzed',
u'type': u'string',
"container": {
"index": "not_analyzed",
"type": "string",
},
u'level': {
u'type': u'long',
"container_segments": {
"type": "string",
},
u'max_level': {
u'type': u'long',
"curr_backup_level": {
"type": "string",
},
u'mode': {
u'type': u'string',
"current_level": {
"type": "string",
},
u'timestamp': {
u'type': u'long',
"dry_run": {
"type": "boolean",
},
u'total_backup_session_size': {
u'type': u'long',
"encrypted": {
"type": "boolean",
},
u'total_broken_links': {
u'type': u'long',
"excluded_files": {
"type": "string",
},
u'total_directories': {
u'type': u'long',
"fs_real_path": {
"type": "string",
},
u'total_fs_files': {
u'type': u'long',
"host_name": {
"index": "not_analyzed",
"type": "string",
},
u'version': {
u'type': u'string',
"hostname": {
"type": "string",
},
u'vol_snap_path': {
u'type': u'string',
"level": {
"type": "long",
},
"max_level": {
"type": "long",
},
"meta_data_file": {
"type": "string",
},
"mode": {
"type": "string",
},
"path_to_backup": {
"type": "string",
},
"time_stamp": {
"type": "string",
},
"timestamp": {
"type": "long",
},
"total_backup_session_size": {
"type": "long",
},
"total_broken_links": {
"type": "long",
},
"total_directories": {
"type": "long",
},
"total_fs_files": {
"type": "long",
},
"version": {
"type": "string",
},
"vol_snap_path": {
"type": "string",
},
},
},
u'user_id': {
u'index': u'not_analyzed',
u'type': u'string',
"user_id": {
"index": "not_analyzed",
"type": "string",
},
u'user_name': {
u'type': u'string',
"user_name": {
"type": "string",
},
},
}
@ -165,6 +205,9 @@ jobs_mapping = {
"dry_run": {
"type": "boolean"
},
"log_file": {
"type": "string"
},
"lvm_auto_snap": {
"type": "string"
},
@ -203,6 +246,9 @@ jobs_mapping = {
},
"restore_abs_path": {
"type": "string"
},
"restore_from_host": {
"type": "string"
}
}
},
@ -279,7 +325,7 @@ jobs_mapping = {
def get_mappings():
return {
u'jobs': jobs_mapping,
u'backups': backups_mapping,
u'clients': clients_mapping
"jobs": jobs_mapping,
"backups": backups_mapping,
"clients": clients_mapping
}

40
tests/test_db_init.py
View File

@ -31,7 +31,8 @@ from freezer_api.cmd.db_init import (ElastichsearchEngine,
parse_config_file,
get_db_params,
main,
DEFAULT_CONF_PATH)
DEFAULT_CONF_PATH,
MergeMappingException)
from freezer_api.common import db_mappings
@ -45,11 +46,15 @@ class TestElasticsearchEngine(unittest.TestCase):
}
self.mock_resp = Mock()
self.mock_args = Mock()
self.mock_args.test_only = False
self.mock_args.always_yes = False
self.mock_args.verbose = 1
self.mock_args.select_mapping = ''
self.mock_args.erase = False
self.es_manager = ElastichsearchEngine(es_url='http://test:9333',
es_index='freezerindex',
test_only=False,
always_yes=False,
verbose=True)
args=self.mock_args)
def test_new(self):
self.assertIsInstance(self.es_manager, ElastichsearchEngine)
@ -72,17 +77,19 @@ class TestElasticsearchEngine(unittest.TestCase):
@patch.object(ElastichsearchEngine, 'proceed')
@patch.object(ElastichsearchEngine, 'delete_type')
@patch.object(ElastichsearchEngine, 'put_mapping')
def test_askput_calls_delete_and_put_mappunts_when_always_yes(self,
def test_askput_calls_delete_and_put_mappings_when_always_yes_and_erase(self,
mock_put_mapping,
mock_delete_type,
mock_proceed):
self.es_manager.always_yes = True
self.mock_args.yes = True
self.mock_args.erase = True
mock_put_mapping.side_effect = [MergeMappingException('regular test failure'), 0]
res = self.es_manager.askput_mapping('jobs', self.test_mappings['jobs'])
self.assertTrue(mock_put_mapping.called)
mock_delete_type.assert_called_once_with('jobs')
mock_put_mapping.assert_called_once_with('jobs', self.test_mappings['jobs'])
def test_askput_does_nothing_when_test_only(self):
self.es_manager.test_only = True
self.mock_args.test_only = True
res = self.es_manager.askput_mapping('jobs', self.test_mappings['jobs'])
self.assertEquals(None, res)
@ -176,8 +183,7 @@ class TestElasticsearchEngine(unittest.TestCase):
_raw_input.assert_called_once_with('are you drunk ?')
def test_proceed_returns_true_when_always_yes(self):
self.es_manager.always_yes = True
res = self.es_manager.proceed('ask me not')
res = self.es_manager.proceed('ask me not', True)
self.assertTrue(res)
@patch('freezer_api.cmd.db_init.requests')
@ -210,12 +216,20 @@ class TestElasticsearchEngine(unittest.TestCase):
class TestDbInit(unittest.TestCase):
def setUp(self):
self.mock_args = Mock()
self.mock_args.test_only = False
self.mock_args.always_yes = False
self.mock_args.verbose = 1
self.mock_args.select_mapping = ''
self.mock_args.erase = False
@patch('freezer_api.cmd.db_init.argparse.ArgumentParser')
def test_get_args_calls_add_argument(self, mock_ArgumentParser):
mock_arg_parser = Mock()
mock_ArgumentParser.return_value = mock_arg_parser
retval = get_args()
retval = get_args([])
call_count = mock_arg_parser.add_argument.call_count
self.assertGreater(call_count, 6)
@ -262,8 +276,11 @@ class TestDbInit(unittest.TestCase):
@patch('freezer_api.cmd.db_init.get_args')
def test_main_calls_esmanager_put_mappings_with_mappings(self, mock_get_args, mock_get_db_params,
mock_ElastichsearchEngine):
mock_get_args.return_value = self.mock_args
mock_get_db_params.return_value = Mock(), Mock()
mock_es_manager = Mock()
mock_es_manager.put_mappings.return_value = os.EX_OK
mock_ElastichsearchEngine.return_value = mock_es_manager
res = main()
@ -276,6 +293,7 @@ class TestDbInit(unittest.TestCase):
@patch('freezer_api.cmd.db_init.get_args')
def test_main_return_EX_DATAERR_exitcode_on_error(self, mock_get_args, mock_get_db_params,
mock_ElastichsearchEngine):
mock_get_args.return_value = self.mock_args
mock_get_db_params.return_value = Mock(), Mock()
mock_es_manager = Mock()
mock_ElastichsearchEngine.return_value = mock_es_manager