Merge "cmd: Deprecate heat-manage migrate_properties_data command"
This commit is contained in:
Zuul
Gerrit Code Review
commit
7668b97777
|
|
@ -146,8 +146,10 @@ def do_crypt_parameters_and_properties():
|
|||
|
||||
|
||||
def do_properties_data_migrate():
|
||||
ctxt = context.get_admin_context()
|
||||
db_api.db_properties_data_migrate(ctxt)
|
||||
print(
|
||||
'This command has been deprecated and is now a no-op. '
|
||||
'It will be removed in a future release.'
|
||||
)
|
||||
|
||||
|
||||
def add_command_parsers(subparsers):
|
||||
|
|
|
|||
316
heat/db/api.py
316
heat/db/api.py
|
|
@ -1782,319 +1782,3 @@ def sync_point_update_input_data(context, entity_id,
|
|||
atomic_key=atomic_key
|
||||
).update({"input_data": input_data, "atomic_key": atomic_key + 1})
|
||||
return rows_updated
|
||||
|
||||
|
||||
# data migration utils
|
||||
|
||||
|
||||
def _crypt_action(encrypt):
|
||||
if encrypt:
|
||||
return _('encrypt')
|
||||
return _('decrypt')
|
||||
|
||||
|
||||
def _db_encrypt_or_decrypt_template_params(
|
||||
context, encryption_key, encrypt=False, batch_size=50, verbose=False,
|
||||
):
|
||||
from heat.engine import template
|
||||
excs = []
|
||||
query = context.session.query(models.RawTemplate)
|
||||
template_batches = _get_batch(
|
||||
context.session, context=context, query=query,
|
||||
model=models.RawTemplate,
|
||||
batch_size=batch_size)
|
||||
next_batch = list(itertools.islice(template_batches, batch_size))
|
||||
while next_batch:
|
||||
with context.session.begin():
|
||||
for raw_template in next_batch:
|
||||
try:
|
||||
if verbose:
|
||||
LOG.info("Processing raw_template %s...",
|
||||
raw_template.id)
|
||||
env = raw_template.environment
|
||||
needs_update = False
|
||||
|
||||
# using "in env.keys()" so an exception is raised
|
||||
# if env is something weird like a string.
|
||||
if env is None or 'parameters' not in env.keys():
|
||||
continue
|
||||
if 'encrypted_param_names' in env:
|
||||
encrypted_params = env['encrypted_param_names']
|
||||
else:
|
||||
encrypted_params = []
|
||||
|
||||
if encrypt:
|
||||
tmpl = template.Template.load(
|
||||
context, raw_template.id, raw_template)
|
||||
param_schemata = tmpl.param_schemata()
|
||||
if not param_schemata:
|
||||
continue
|
||||
|
||||
for param_name, param_val in env['parameters'].items():
|
||||
if (param_name in encrypted_params or
|
||||
param_name not in param_schemata or
|
||||
not param_schemata[param_name].hidden):
|
||||
continue
|
||||
encrypted_val = crypt.encrypt(
|
||||
str(param_val), encryption_key)
|
||||
env['parameters'][param_name] = encrypted_val
|
||||
encrypted_params.append(param_name)
|
||||
needs_update = True
|
||||
if needs_update:
|
||||
newenv = env.copy()
|
||||
newenv['encrypted_param_names'] = encrypted_params
|
||||
else: # decrypt
|
||||
for param_name in encrypted_params:
|
||||
method, value = env['parameters'][param_name]
|
||||
decrypted_val = crypt.decrypt(method, value,
|
||||
encryption_key)
|
||||
env['parameters'][param_name] = decrypted_val
|
||||
needs_update = True
|
||||
if needs_update:
|
||||
newenv = env.copy()
|
||||
newenv['encrypted_param_names'] = []
|
||||
|
||||
if needs_update:
|
||||
raw_template_update(context, raw_template.id,
|
||||
{'environment': newenv})
|
||||
except Exception as exc:
|
||||
LOG.exception('Failed to %(crypt_action)s parameters '
|
||||
'of raw template %(id)d',
|
||||
{'id': raw_template.id,
|
||||
'crypt_action': _crypt_action(encrypt)})
|
||||
excs.append(exc)
|
||||
continue
|
||||
finally:
|
||||
if verbose:
|
||||
LOG.info("Finished %(crypt_action)s processing of "
|
||||
"raw_template %(id)d.",
|
||||
{'id': raw_template.id,
|
||||
'crypt_action': _crypt_action(encrypt)})
|
||||
next_batch = list(itertools.islice(template_batches, batch_size))
|
||||
return excs
|
||||
|
||||
|
||||
def _db_encrypt_or_decrypt_resource_prop_data_legacy(
|
||||
context, encryption_key, encrypt=False, batch_size=50, verbose=False,
|
||||
):
|
||||
excs = []
|
||||
|
||||
# Older resources may have properties_data in the legacy column,
|
||||
# so update those as needed
|
||||
query = context.session.query(models.Resource).filter(
|
||||
models.Resource.properties_data_encrypted.isnot(encrypt))
|
||||
resource_batches = _get_batch(
|
||||
session=context.session, context=context, query=query,
|
||||
model=models.Resource,
|
||||
batch_size=batch_size)
|
||||
next_batch = list(itertools.islice(resource_batches, batch_size))
|
||||
while next_batch:
|
||||
with context.session.begin():
|
||||
for resource in next_batch:
|
||||
if not resource.properties_data:
|
||||
continue
|
||||
try:
|
||||
if verbose:
|
||||
LOG.info("Processing resource %s...",
|
||||
resource.id)
|
||||
if encrypt:
|
||||
result = crypt.encrypted_dict(resource.properties_data,
|
||||
encryption_key)
|
||||
else:
|
||||
result = crypt.decrypted_dict(resource.properties_data,
|
||||
encryption_key)
|
||||
_try_resource_update(
|
||||
context, resource.id,
|
||||
{'properties_data': result,
|
||||
'properties_data_encrypted': encrypt},
|
||||
resource.atomic_key)
|
||||
except Exception as exc:
|
||||
LOG.exception('Failed to %(crypt_action)s '
|
||||
'properties_data of resource %(id)d' %
|
||||
{'id': resource.id,
|
||||
'crypt_action': _crypt_action(encrypt)})
|
||||
excs.append(exc)
|
||||
continue
|
||||
finally:
|
||||
if verbose:
|
||||
LOG.info("Finished processing resource %s.",
|
||||
resource.id)
|
||||
next_batch = list(itertools.islice(resource_batches, batch_size))
|
||||
return excs
|
||||
|
||||
|
||||
def _db_encrypt_or_decrypt_resource_prop_data(
|
||||
context, encryption_key, encrypt=False, batch_size=50, verbose=False,
|
||||
):
|
||||
excs = []
|
||||
|
||||
# Older resources may have properties_data in the legacy column,
|
||||
# so update those as needed
|
||||
query = context.session.query(models.ResourcePropertiesData).filter(
|
||||
models.ResourcePropertiesData.encrypted.isnot(encrypt))
|
||||
rpd_batches = _get_batch(
|
||||
session=context.session, context=context, query=query,
|
||||
model=models.ResourcePropertiesData, batch_size=batch_size)
|
||||
next_batch = list(itertools.islice(rpd_batches, batch_size))
|
||||
while next_batch:
|
||||
with context.session.begin():
|
||||
for rpd in next_batch:
|
||||
if not rpd.data:
|
||||
continue
|
||||
try:
|
||||
if verbose:
|
||||
LOG.info("Processing resource_properties_data "
|
||||
"%s...", rpd.id)
|
||||
if encrypt:
|
||||
result = crypt.encrypted_dict(rpd.data,
|
||||
encryption_key)
|
||||
else:
|
||||
result = crypt.decrypted_dict(rpd.data,
|
||||
encryption_key)
|
||||
rpd.update({'data': result,
|
||||
'encrypted': encrypt})
|
||||
except Exception as exc:
|
||||
LOG.exception(
|
||||
"Failed to %(crypt_action)s "
|
||||
"data of resource_properties_data %(id)d" %
|
||||
{'id': rpd.id,
|
||||
'crypt_action': _crypt_action(encrypt)})
|
||||
excs.append(exc)
|
||||
continue
|
||||
finally:
|
||||
if verbose:
|
||||
LOG.info(
|
||||
"Finished processing resource_properties_data"
|
||||
" %s.", rpd.id)
|
||||
next_batch = list(itertools.islice(rpd_batches, batch_size))
|
||||
return excs
|
||||
|
||||
|
||||
def db_encrypt_parameters_and_properties(
|
||||
context, encryption_key, batch_size=50, verbose=False,
|
||||
):
|
||||
"""Encrypt parameters and properties for all templates in db.
|
||||
|
||||
:param context: RPC context
|
||||
:param encryption_key: key that will be used for parameter and property
|
||||
encryption
|
||||
:param batch_size: number of templates requested from DB in each iteration.
|
||||
50 means that heat requests 50 templates, encrypt them
|
||||
and proceed with next 50 items.
|
||||
:param verbose: log an INFO message when processing of each raw_template or
|
||||
resource begins or ends
|
||||
:return: list of exceptions encountered during encryption
|
||||
"""
|
||||
excs = []
|
||||
excs.extend(_db_encrypt_or_decrypt_template_params(
|
||||
context, encryption_key, True, batch_size, verbose))
|
||||
excs.extend(_db_encrypt_or_decrypt_resource_prop_data(
|
||||
context, encryption_key, True, batch_size, verbose))
|
||||
excs.extend(_db_encrypt_or_decrypt_resource_prop_data_legacy(
|
||||
context, encryption_key, True, batch_size, verbose))
|
||||
return excs
|
||||
|
||||
|
||||
def db_decrypt_parameters_and_properties(
|
||||
context, encryption_key, batch_size=50, verbose=False,
|
||||
):
|
||||
"""Decrypt parameters and properties for all templates in db.
|
||||
|
||||
:param context: RPC context
|
||||
:param encryption_key: key that will be used for parameter and property
|
||||
decryption
|
||||
:param batch_size: number of templates requested from DB in each iteration.
|
||||
50 means that heat requests 50 templates, encrypt them
|
||||
and proceed with next 50 items.
|
||||
:param verbose: log an INFO message when processing of each raw_template or
|
||||
resource begins or ends
|
||||
:return: list of exceptions encountered during decryption
|
||||
"""
|
||||
excs = []
|
||||
excs.extend(_db_encrypt_or_decrypt_template_params(
|
||||
context, encryption_key, False, batch_size, verbose))
|
||||
excs.extend(_db_encrypt_or_decrypt_resource_prop_data(
|
||||
context, encryption_key, False, batch_size, verbose))
|
||||
excs.extend(_db_encrypt_or_decrypt_resource_prop_data_legacy(
|
||||
context, encryption_key, False, batch_size, verbose))
|
||||
return excs
|
||||
|
||||
|
||||
def db_properties_data_migrate(context, batch_size=50):
|
||||
"""Migrate properties data from legacy columns to new location in db.
|
||||
|
||||
:param context: RPC context
|
||||
:param batch_size: number of templates requested from DB in each iteration.
|
||||
50 means that heat requests 50 templates, encrypt them
|
||||
and proceed with next 50 items.
|
||||
"""
|
||||
|
||||
query = context.session.query(models.Resource).filter(and_(
|
||||
models.Resource.properties_data.isnot(None),
|
||||
models.Resource.rsrc_prop_data_id.is_(None)))
|
||||
resource_batches = _get_batch(
|
||||
session=context.session, context=context, query=query,
|
||||
model=models.Resource, batch_size=batch_size)
|
||||
next_batch = list(itertools.islice(resource_batches, batch_size))
|
||||
while next_batch:
|
||||
with context.session.begin():
|
||||
for resource in next_batch:
|
||||
try:
|
||||
encrypted = resource.properties_data_encrypted
|
||||
if encrypted is None:
|
||||
LOG.warning(
|
||||
'Unexpected: resource.encrypted is None for '
|
||||
'resource id %s for legacy '
|
||||
'resource.properties_data, assuming False.',
|
||||
resource.id)
|
||||
encrypted = False
|
||||
rsrc_prop_data = resource_prop_data_create(
|
||||
context, {'encrypted': encrypted,
|
||||
'data': resource.properties_data})
|
||||
resource_update(context, resource.id,
|
||||
{'properties_data_encrypted': None,
|
||||
'properties_data': None,
|
||||
'rsrc_prop_data_id': rsrc_prop_data.id},
|
||||
resource.atomic_key)
|
||||
except Exception:
|
||||
LOG.exception('Failed to migrate properties_data for '
|
||||
'resource %d', resource.id)
|
||||
continue
|
||||
next_batch = list(itertools.islice(resource_batches, batch_size))
|
||||
|
||||
query = context.session.query(models.Event).filter(and_(
|
||||
models.Event.resource_properties.isnot(None),
|
||||
models.Event.rsrc_prop_data_id.is_(None)))
|
||||
event_batches = _get_batch(
|
||||
session=context.session, context=context, query=query,
|
||||
model=models.Event, batch_size=batch_size)
|
||||
next_batch = list(itertools.islice(event_batches, batch_size))
|
||||
while next_batch:
|
||||
with context.session.begin():
|
||||
for event in next_batch:
|
||||
try:
|
||||
prop_data = event.resource_properties
|
||||
rsrc_prop_data = resource_prop_data_create(
|
||||
context,
|
||||
{'encrypted': False, 'data': prop_data})
|
||||
event.update({'resource_properties': None,
|
||||
'rsrc_prop_data_id': rsrc_prop_data.id})
|
||||
except Exception:
|
||||
LOG.exception('Failed to migrate resource_properties '
|
||||
'for event %d', event.id)
|
||||
continue
|
||||
next_batch = list(itertools.islice(event_batches, batch_size))
|
||||
|
||||
|
||||
def _get_batch(session, context, query, model, batch_size=50):
|
||||
last_batch_marker = None
|
||||
while True:
|
||||
results = _paginate_query(
|
||||
context=context, query=query, model=model, limit=batch_size,
|
||||
marker=last_batch_marker).all()
|
||||
if not results:
|
||||
break
|
||||
else:
|
||||
for result in results:
|
||||
yield result
|
||||
last_batch_marker = results[-1].id
|
||||
|
|
|
|||
|
|
@ -11,15 +11,12 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import copy
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from unittest import mock
|
||||
import uuid
|
||||
|
||||
import fixtures
|
||||
from oslo_config import cfg
|
||||
from oslo_db import exception as db_exception
|
||||
from oslo_utils import timeutils
|
||||
|
|
@ -3376,536 +3373,6 @@ class DBAPISyncPointTest(common.HeatTestCase):
|
|||
self.assertEqual(len(self.resources) * 21, add.call_count)
|
||||
|
||||
|
||||
class DBAPIMigratePropertiesDataTest(common.HeatTestCase):
|
||||
def setUp(self):
|
||||
super(DBAPIMigratePropertiesDataTest, self).setUp()
|
||||
self.ctx = utils.dummy_context()
|
||||
templ = create_raw_template(self.ctx)
|
||||
user_creds = create_user_creds(self.ctx)
|
||||
stack = create_stack(self.ctx, templ, user_creds)
|
||||
stack2 = create_stack(self.ctx, templ, user_creds)
|
||||
create_resource(self.ctx, stack, True, name='res1')
|
||||
create_resource(self.ctx, stack2, True, name='res2')
|
||||
create_event(self.ctx, True)
|
||||
create_event(self.ctx, True)
|
||||
|
||||
def _test_migrate_resource(self, batch_size=50):
|
||||
resources = self.ctx.session.query(models.Resource).all()
|
||||
self.assertEqual(2, len(resources))
|
||||
for resource in resources:
|
||||
self.assertEqual('bar1', resource.properties_data['foo1'])
|
||||
|
||||
db_api.db_properties_data_migrate(self.ctx, batch_size=batch_size)
|
||||
for resource in resources:
|
||||
self.assertEqual('bar1', resource.rsrc_prop_data.data['foo1'])
|
||||
self.assertFalse(resource.rsrc_prop_data.encrypted)
|
||||
self.assertIsNone(resource.properties_data)
|
||||
self.assertIsNone(resource.properties_data_encrypted)
|
||||
|
||||
def _test_migrate_event(self, batch_size=50):
|
||||
events = self.ctx.session.query(models.Event).all()
|
||||
self.assertEqual(2, len(events))
|
||||
for event in events:
|
||||
self.assertEqual('ev_bar', event.resource_properties['foo2'])
|
||||
|
||||
db_api.db_properties_data_migrate(self.ctx, batch_size=batch_size)
|
||||
self.ctx.session.expire_all()
|
||||
events = self.ctx.session.query(models.Event).all()
|
||||
for event in events:
|
||||
self.assertEqual('ev_bar', event.rsrc_prop_data.data['foo2'])
|
||||
self.assertFalse(event.rsrc_prop_data.encrypted)
|
||||
self.assertIsNone(event.resource_properties)
|
||||
|
||||
def test_migrate_event(self):
|
||||
self._test_migrate_event()
|
||||
|
||||
def test_migrate_event_in_batches(self):
|
||||
self._test_migrate_event(batch_size=1)
|
||||
|
||||
def test_migrate_resource(self):
|
||||
self._test_migrate_resource()
|
||||
|
||||
def test_migrate_resource_in_batches(self):
|
||||
self._test_migrate_resource(batch_size=1)
|
||||
|
||||
def test_migrate_encrypted_resource(self):
|
||||
resources = self.ctx.session.query(models.Resource).all()
|
||||
db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, 'i have a key for you if you want')
|
||||
|
||||
encrypted_data_pre_migration = resources[0].properties_data['foo1'][1]
|
||||
db_api.db_properties_data_migrate(self.ctx)
|
||||
resources = self.ctx.session.query(models.Resource).all()
|
||||
|
||||
self.assertTrue(resources[0].rsrc_prop_data.encrypted)
|
||||
self.assertIsNone(resources[0].properties_data)
|
||||
self.assertIsNone(resources[0].properties_data_encrypted)
|
||||
self.assertEqual('cryptography_decrypt_v1',
|
||||
resources[0].rsrc_prop_data.data['foo1'][0])
|
||||
self.assertEqual(encrypted_data_pre_migration,
|
||||
resources[0].rsrc_prop_data.data['foo1'][1])
|
||||
|
||||
db_api.db_decrypt_parameters_and_properties(
|
||||
self.ctx, 'i have a key for you if you want')
|
||||
self.ctx.session.expire_all()
|
||||
resources = self.ctx.session.query(models.Resource).all()
|
||||
|
||||
self.assertEqual('bar1', resources[0].rsrc_prop_data.data['foo1'])
|
||||
self.assertFalse(resources[0].rsrc_prop_data.encrypted)
|
||||
self.assertIsNone(resources[0].properties_data)
|
||||
self.assertIsNone(resources[0].properties_data_encrypted)
|
||||
|
||||
|
||||
class DBAPICryptParamsPropsTest(common.HeatTestCase):
|
||||
def setUp(self):
|
||||
super(DBAPICryptParamsPropsTest, self).setUp()
|
||||
self.ctx = utils.dummy_context()
|
||||
self.template = self._create_template()
|
||||
self.user_creds = create_user_creds(self.ctx)
|
||||
self.stack = create_stack(self.ctx, self.template, self.user_creds)
|
||||
self.resources = [create_resource(self.ctx, self.stack, name='res1')]
|
||||
|
||||
hidden_params_dict = {
|
||||
'param2': 'bar',
|
||||
'param_number': '456',
|
||||
'param_boolean': '1',
|
||||
'param_map': '{\"test\":\"json\"}',
|
||||
'param_comma_list': '[\"Hola\", \"Senor\"]'}
|
||||
|
||||
def _create_template(self):
|
||||
"""Initialize sample template."""
|
||||
self.t = template_format.parse('''
|
||||
heat_template_version: 2013-05-23
|
||||
parameters:
|
||||
param1:
|
||||
type: string
|
||||
description: value1.
|
||||
param2:
|
||||
type: string
|
||||
description: value2.
|
||||
hidden: true
|
||||
param3:
|
||||
type: string
|
||||
description: value3
|
||||
hidden: true
|
||||
default: "don't encrypt me! I'm not sensitive enough"
|
||||
param_string_default_int:
|
||||
type: string
|
||||
description: String parameter with integer default value
|
||||
default: 4353
|
||||
hidden: true
|
||||
param_number:
|
||||
type: number
|
||||
description: Number parameter
|
||||
default: 4353
|
||||
hidden: true
|
||||
param_boolean:
|
||||
type: boolean
|
||||
description: boolean parameter
|
||||
default: true
|
||||
hidden: true
|
||||
param_map:
|
||||
type: json
|
||||
description: json parameter
|
||||
default: {"fee": {"fi":"fo"}}
|
||||
hidden: true
|
||||
param_comma_list:
|
||||
type: comma_delimited_list
|
||||
description: cdl parameter
|
||||
default: ["hola", "senorita"]
|
||||
hidden: true
|
||||
resources:
|
||||
a_resource:
|
||||
type: GenericResourceType
|
||||
''')
|
||||
template = {
|
||||
'template': self.t,
|
||||
'files': {'foo': 'bar'},
|
||||
'environment': {
|
||||
'parameters': {
|
||||
'param1': 'foo',
|
||||
'param2': 'bar',
|
||||
'param_number': '456',
|
||||
'param_boolean': '1',
|
||||
'param_map': '{\"test\":\"json\"}',
|
||||
'param_comma_list': '[\"Hola\", \"Senor\"]'}}}
|
||||
|
||||
return db_api.raw_template_create(self.ctx, template)
|
||||
|
||||
def encrypt(self, enc_key=None, batch_size=50,
|
||||
legacy_prop_data=False):
|
||||
session = self.ctx.session
|
||||
if enc_key is None:
|
||||
enc_key = cfg.CONF.auth_encryption_key
|
||||
self.assertEqual([], db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, enc_key, batch_size=batch_size))
|
||||
for enc_tmpl in session.query(models.RawTemplate).all():
|
||||
for param_name in self.hidden_params_dict.keys():
|
||||
self.assertEqual(
|
||||
'cryptography_decrypt_v1',
|
||||
enc_tmpl.environment['parameters'][param_name][0])
|
||||
self.assertEqual(
|
||||
'foo', enc_tmpl.environment['parameters']['param1'])
|
||||
# test that decryption does not store (or encrypt) default
|
||||
# values in template's environment['parameters']
|
||||
self.assertIsNone(
|
||||
enc_tmpl.environment['parameters'].get('param3'))
|
||||
|
||||
enc_resources = session.query(models.Resource).all()
|
||||
self.assertNotEqual([], enc_resources)
|
||||
for enc_resource in enc_resources:
|
||||
if legacy_prop_data:
|
||||
self.assertEqual(
|
||||
'cryptography_decrypt_v1',
|
||||
enc_resource.properties_data['foo1'][0])
|
||||
else:
|
||||
self.assertEqual(
|
||||
'cryptography_decrypt_v1',
|
||||
enc_resource.rsrc_prop_data.data['foo1'][0])
|
||||
|
||||
ev = enc_tmpl.environment['parameters']['param2'][1]
|
||||
return ev
|
||||
|
||||
def decrypt(self, encrypt_value, enc_key=None,
|
||||
batch_size=50, legacy_prop_data=False):
|
||||
session = self.ctx.session
|
||||
if enc_key is None:
|
||||
enc_key = cfg.CONF.auth_encryption_key
|
||||
|
||||
self.assertEqual([], db_api.db_decrypt_parameters_and_properties(
|
||||
self.ctx, enc_key, batch_size=batch_size))
|
||||
|
||||
for dec_tmpl in session.query(models.RawTemplate).all():
|
||||
self.assertNotEqual(
|
||||
encrypt_value,
|
||||
dec_tmpl.environment['parameters']['param2'][1])
|
||||
for param_name, param_value in self.hidden_params_dict.items():
|
||||
self.assertEqual(
|
||||
param_value,
|
||||
dec_tmpl.environment['parameters'][param_name])
|
||||
self.assertEqual(
|
||||
'foo', dec_tmpl.environment['parameters']['param1'])
|
||||
self.assertIsNone(
|
||||
dec_tmpl.environment['parameters'].get('param3'))
|
||||
|
||||
# test that decryption does not store default
|
||||
# values in template's environment['parameters']
|
||||
self.assertIsNone(dec_tmpl.environment['parameters'].get(
|
||||
'param3'))
|
||||
|
||||
decrypt_value = dec_tmpl.environment['parameters']['param2'][1]
|
||||
|
||||
dec_resources = session.query(models.Resource).all()
|
||||
self.assertNotEqual([], dec_resources)
|
||||
for dec_resource in dec_resources:
|
||||
if legacy_prop_data:
|
||||
self.assertEqual(
|
||||
'bar1', dec_resource.properties_data['foo1'])
|
||||
else:
|
||||
self.assertEqual(
|
||||
'bar1', dec_resource.rsrc_prop_data.data['foo1'])
|
||||
|
||||
return decrypt_value
|
||||
|
||||
def _test_db_encrypt_decrypt(self, batch_size=50, legacy_prop_data=False):
|
||||
session = self.ctx.session
|
||||
raw_templates = session.query(models.RawTemplate).all()
|
||||
self.assertNotEqual([], raw_templates)
|
||||
for r_tmpl in raw_templates:
|
||||
for param_name, param_value in self.hidden_params_dict.items():
|
||||
self.assertEqual(param_value,
|
||||
r_tmpl.environment['parameters'][param_name])
|
||||
self.assertEqual('foo',
|
||||
r_tmpl.environment['parameters']['param1'])
|
||||
|
||||
resources = session.query(models.Resource).all()
|
||||
self.assertNotEqual([], resources)
|
||||
self.assertEqual(len(resources), len(raw_templates))
|
||||
for resource in resources:
|
||||
resource = db_api.resource_get(self.ctx, resource.id)
|
||||
if legacy_prop_data:
|
||||
self.assertEqual(
|
||||
'bar1', resource.properties_data['foo1'])
|
||||
else:
|
||||
self.assertEqual(
|
||||
'bar1', resource.rsrc_prop_data.data['foo1'])
|
||||
|
||||
# Test encryption
|
||||
encrypt_value = self.encrypt(batch_size=batch_size,
|
||||
legacy_prop_data=legacy_prop_data)
|
||||
|
||||
# Test that encryption is idempotent
|
||||
encrypt_value2 = self.encrypt(batch_size=batch_size,
|
||||
legacy_prop_data=legacy_prop_data)
|
||||
self.assertEqual(encrypt_value, encrypt_value2)
|
||||
|
||||
# Test decryption
|
||||
decrypt_value = self.decrypt(encrypt_value, batch_size=batch_size,
|
||||
legacy_prop_data=legacy_prop_data)
|
||||
|
||||
# Test that decryption is idempotent
|
||||
decrypt_value2 = self.decrypt(encrypt_value, batch_size=batch_size,
|
||||
legacy_prop_data=legacy_prop_data)
|
||||
self.assertEqual(decrypt_value, decrypt_value2)
|
||||
|
||||
# Test using a different encryption key to encrypt & decrypt
|
||||
encrypt_value3 = self.encrypt(
|
||||
enc_key='774c15be099ea74123a9b9592ff12680',
|
||||
batch_size=batch_size, legacy_prop_data=legacy_prop_data)
|
||||
decrypt_value3 = self.decrypt(
|
||||
encrypt_value3, enc_key='774c15be099ea74123a9b9592ff12680',
|
||||
batch_size=batch_size, legacy_prop_data=legacy_prop_data)
|
||||
self.assertEqual(decrypt_value, decrypt_value3)
|
||||
self.assertNotEqual(encrypt_value, decrypt_value)
|
||||
self.assertNotEqual(encrypt_value3, decrypt_value3)
|
||||
self.assertNotEqual(encrypt_value, encrypt_value3)
|
||||
|
||||
def test_db_encrypt_decrypt(self):
|
||||
"""Test encryption and decryption for single template and resource."""
|
||||
self._test_db_encrypt_decrypt()
|
||||
|
||||
def test_db_encrypt_decrypt_legacy_prop_data(self):
|
||||
"""Test encryption and decryption for res with legacy prop data."""
|
||||
# delete what setUp created
|
||||
[self.ctx.session.delete(r) for r in
|
||||
self.ctx.session.query(models.Resource).all()]
|
||||
[self.ctx.session.delete(s) for s in
|
||||
self.ctx.session.query(models.Stack).all()]
|
||||
[self.ctx.session.delete(t) for t in
|
||||
self.ctx.session.query(models.RawTemplate).all()]
|
||||
|
||||
tmpl = self._create_template()
|
||||
stack = create_stack(self.ctx, tmpl, self.user_creds)
|
||||
create_resource(self.ctx, stack, True, name='res1')
|
||||
self._test_db_encrypt_decrypt(legacy_prop_data=True)
|
||||
|
||||
def test_db_encrypt_decrypt_in_batches(self):
|
||||
"""Test encryption and decryption in for several templates and resources.
|
||||
|
||||
Test encryption and decryption with set batch size of
|
||||
templates and resources.
|
||||
"""
|
||||
tmpl1 = self._create_template()
|
||||
tmpl2 = self._create_template()
|
||||
stack = create_stack(self.ctx, tmpl1, self.user_creds)
|
||||
create_resource(self.ctx, stack, False, name='res1')
|
||||
stack2 = create_stack(self.ctx, tmpl2, self.user_creds)
|
||||
create_resource(self.ctx, stack2, False, name='res2')
|
||||
|
||||
self._test_db_encrypt_decrypt(batch_size=1)
|
||||
|
||||
def test_db_encrypt_decrypt_exception_continue(self):
|
||||
"""Test that encryption and decryption proceed after an exception"""
|
||||
def create_malformed_template():
|
||||
"""Initialize a malformed template which should fail encryption."""
|
||||
t = template_format.parse('''
|
||||
heat_template_version: 2013-05-23
|
||||
parameters:
|
||||
param1:
|
||||
type: string
|
||||
description: value1.
|
||||
param2:
|
||||
type: string
|
||||
description: value2.
|
||||
hidden: true
|
||||
param3:
|
||||
type: string
|
||||
description: value3
|
||||
hidden: true
|
||||
default: "don't encrypt me! I'm not sensitive enough"
|
||||
resources:
|
||||
a_resource:
|
||||
type: GenericResourceType
|
||||
''')
|
||||
template = {
|
||||
'template': t,
|
||||
'files': {'foo': 'bar'},
|
||||
'environment': ''} # <- environment should be a dict
|
||||
|
||||
return db_api.raw_template_create(self.ctx, template)
|
||||
|
||||
create_malformed_template()
|
||||
self._create_template()
|
||||
|
||||
# Test encryption
|
||||
enc_result = db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key, batch_size=50)
|
||||
self.assertEqual(1, len(enc_result))
|
||||
self.assertIs(AttributeError, type(enc_result[0]))
|
||||
enc_tmpls = self.ctx.session.query(models.RawTemplate).all()
|
||||
self.assertEqual('', enc_tmpls[1].environment)
|
||||
self.assertEqual('cryptography_decrypt_v1',
|
||||
enc_tmpls[2].environment['parameters']['param2'][0])
|
||||
|
||||
# Test decryption
|
||||
dec_result = db_api.db_decrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key, batch_size=50)
|
||||
self.assertEqual(len(dec_result), 1)
|
||||
self.assertIs(AttributeError, type(dec_result[0]))
|
||||
dec_tmpls = self.ctx.session.query(models.RawTemplate).all()
|
||||
self.assertEqual('', dec_tmpls[1].environment)
|
||||
self.assertEqual('bar',
|
||||
dec_tmpls[2].environment['parameters']['param2'])
|
||||
|
||||
def test_db_encrypt_no_env(self):
|
||||
template = {
|
||||
'template': self.t,
|
||||
'files': {'foo': 'bar'},
|
||||
'environment': None}
|
||||
db_api.raw_template_create(self.ctx, template)
|
||||
self.assertEqual([], db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key))
|
||||
|
||||
def test_db_encrypt_no_env_parameters(self):
|
||||
template = {
|
||||
'template': self.t,
|
||||
'files': {'foo': 'bar'},
|
||||
'environment': {'encrypted_param_names': ['a']}}
|
||||
db_api.raw_template_create(self.ctx, template)
|
||||
self.assertEqual([], db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key))
|
||||
|
||||
def test_db_encrypt_no_properties_data(self):
|
||||
ctx = utils.dummy_context()
|
||||
template = self._create_template()
|
||||
user_creds = create_user_creds(ctx)
|
||||
stack = create_stack(ctx, template, user_creds)
|
||||
resources = [create_resource(ctx, stack, name='res1')]
|
||||
resources[0].properties_data = None
|
||||
self.assertEqual([], db_api.db_encrypt_parameters_and_properties(
|
||||
ctx, cfg.CONF.auth_encryption_key))
|
||||
|
||||
def test_db_encrypt_decrypt_verbose_on(self):
|
||||
info_logger = self.useFixture(
|
||||
fixtures.FakeLogger(level=logging.INFO,
|
||||
format="%(levelname)8s [%(name)s] "
|
||||
"%(message)s"))
|
||||
ctx = utils.dummy_context()
|
||||
template = self._create_template()
|
||||
user_creds = create_user_creds(ctx)
|
||||
stack = create_stack(ctx, template, user_creds)
|
||||
create_resource(ctx, stack, legacy_prop_data=True, name='res2')
|
||||
|
||||
db_api.db_encrypt_parameters_and_properties(
|
||||
ctx, cfg.CONF.auth_encryption_key, verbose=True)
|
||||
self.assertIn("Processing raw_template 1", info_logger.output)
|
||||
self.assertIn("Finished encrypt processing of raw_template 1",
|
||||
info_logger.output)
|
||||
self.assertIn("Processing resource_properties_data 1",
|
||||
info_logger.output)
|
||||
self.assertIn("Finished processing resource_properties_data 1",
|
||||
info_logger.output)
|
||||
# only the resource with legacy properties data is processed
|
||||
self.assertIn("Processing resource 2", info_logger.output)
|
||||
self.assertIn("Finished processing resource 2", info_logger.output)
|
||||
|
||||
info_logger2 = self.useFixture(
|
||||
fixtures.FakeLogger(level=logging.INFO,
|
||||
format="%(levelname)8s [%(name)s] "
|
||||
"%(message)s"))
|
||||
|
||||
db_api.db_decrypt_parameters_and_properties(
|
||||
ctx, cfg.CONF.auth_encryption_key, verbose=True)
|
||||
self.assertIn("Processing raw_template 1", info_logger2.output)
|
||||
self.assertIn("Finished decrypt processing of raw_template 1",
|
||||
info_logger2.output)
|
||||
self.assertIn("Processing resource_properties_data 1",
|
||||
info_logger.output)
|
||||
self.assertIn("Finished processing resource_properties_data 1",
|
||||
info_logger.output)
|
||||
# only the resource with legacy properties data is processed
|
||||
self.assertIn("Processing resource 2", info_logger2.output)
|
||||
self.assertIn("Finished processing resource 2", info_logger2.output)
|
||||
|
||||
def test_db_encrypt_decrypt_verbose_off(self):
|
||||
info_logger = self.useFixture(
|
||||
fixtures.FakeLogger(level=logging.INFO,
|
||||
format="%(levelname)8s [%(name)s] "
|
||||
"%(message)s"))
|
||||
ctx = utils.dummy_context()
|
||||
template = self._create_template()
|
||||
user_creds = create_user_creds(ctx)
|
||||
stack = create_stack(ctx, template, user_creds)
|
||||
create_resource(ctx, stack, name='res1')
|
||||
|
||||
db_api.db_encrypt_parameters_and_properties(
|
||||
ctx, cfg.CONF.auth_encryption_key, verbose=False)
|
||||
self.assertNotIn("Processing raw_template 1", info_logger.output)
|
||||
self.assertNotIn("Processing resource 1", info_logger.output)
|
||||
self.assertNotIn("Successfully processed raw_template 1",
|
||||
info_logger.output)
|
||||
self.assertNotIn("Successfully processed resource 1",
|
||||
info_logger.output)
|
||||
|
||||
info_logger2 = self.useFixture(
|
||||
fixtures.FakeLogger(level=logging.INFO,
|
||||
format="%(levelname)8s [%(name)s] "
|
||||
"%(message)s"))
|
||||
|
||||
db_api.db_decrypt_parameters_and_properties(
|
||||
ctx, cfg.CONF.auth_encryption_key, verbose=False)
|
||||
self.assertNotIn("Processing raw_template 1", info_logger2.output)
|
||||
self.assertNotIn("Processing resource 1", info_logger2.output)
|
||||
self.assertNotIn("Successfully processed raw_template 1",
|
||||
info_logger2.output)
|
||||
self.assertNotIn("Successfully processed resource 1",
|
||||
info_logger2.output)
|
||||
|
||||
def test_db_encrypt_no_param_schema(self):
|
||||
t = copy.deepcopy(self.t)
|
||||
del(t['parameters']['param2'])
|
||||
template = {
|
||||
'template': t,
|
||||
'files': {'foo': 'bar'},
|
||||
'environment': {'encrypted_param_names': [],
|
||||
'parameters': {'param2': 'foo'}}}
|
||||
db_api.raw_template_create(self.ctx, template)
|
||||
self.assertEqual([], db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key))
|
||||
|
||||
def test_db_encrypt_non_string_param_type(self):
|
||||
t = template_format.parse('''
|
||||
heat_template_version: 2013-05-23
|
||||
parameters:
|
||||
param1:
|
||||
type: string
|
||||
description: value1.
|
||||
param2:
|
||||
type: string
|
||||
description: value2.
|
||||
hidden: true
|
||||
param3:
|
||||
type: string
|
||||
description: value3
|
||||
hidden: true
|
||||
default: 1234
|
||||
resources:
|
||||
a_resource:
|
||||
type: GenericResourceType
|
||||
''')
|
||||
template = {
|
||||
'template': t,
|
||||
'files': {},
|
||||
'environment': {'parameters': {
|
||||
'param1': 'foo',
|
||||
'param2': 'bar',
|
||||
'param3': 12345}}}
|
||||
tmpl = db_api.raw_template_create(self.ctx, template)
|
||||
self.assertEqual([], db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key))
|
||||
tmpl = db_api.raw_template_get(self.ctx, tmpl.id)
|
||||
enc_params = copy.copy(tmpl.environment['parameters'])
|
||||
|
||||
self.assertEqual([], db_api.db_decrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key, batch_size=50))
|
||||
tmpl = db_api.raw_template_get(self.ctx, tmpl.id)
|
||||
dec_params = tmpl.environment['parameters']
|
||||
|
||||
self.assertNotEqual(enc_params['param3'], dec_params['param3'])
|
||||
self.assertEqual('bar', dec_params['param2'])
|
||||
self.assertEqual('12345', dec_params['param3'])
|
||||
|
||||
|
||||
class ResetStackStatusTests(common.HeatTestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
upgrade:
|
||||
- |
|
||||
The ``heat-manage migrate_properties_data`` command is deprecated and is
|
||||
now a no-op. It will be removed in a future release.
|
||||
Loading…
Reference in New Issue