openstack
/
tripleo-common
Code Issues Proposed changes

Don't set deployments as hostvars 

This patch updates config-download to not create a vars file under
group_vars for the server deployments. These variables end up as
hostvars in Ansible, which has caused many escaping problems since
hostvars are run through the templating engine.

In combination with I06c9b3256567cc57d599bd4d9af617c718d2314f, these
escaping problems should be eliminated.

Additionally, by not using hostvars (set_fact), we can save memory since
a hostvar is set on a host and persisted in memory. With many hosts and
large deployments (such as all hiera data), this would have eventually
been an issue.

Instead, an Ansible lookup is used in deployments.yaml to read
deployment data, which are now written to individual files by
config-download. This also improves readability instead of having one
big file of all deployment data.

Change-Id: Ie09fc64cf85eb532f31684f1c9808b942672ff41
Closes-Bug: #1760990
Closes-Bug: #1763779
(cherry picked from commit 60f25d2bcd)
This commit is contained in:
James Slagle 2018-04-06 17:24:08 -04:00
parent 5ce4c109a2
commit 21a8c7f905
22 changed files with 343 additions and 278 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
tripleo_common/templates/group_var_server.j2 → tripleo_common/templates/deployment.j2
View File

@ -1,6 +1,5 @@
deploy_server_id: {{ server_id }}
{% for deployment in deployments %}
{{ deployment.get('deployment_name') }}:
{% if deployment.get('scalar') %}
config: |
@ -28,5 +27,3 @@ deploy_server_id: {{ server_id }}
description: {{ output.get('description') }}
type: {{ output.get('type') }}
{% endfor %}
{% endfor %}

14
tripleo_common/templates/deployments.yaml
View File

@ -1,14 +1,16 @@
- name: Lookup deployment UUID
set_fact:
deployment_uuid: "{{ lookup('file', role_name ~ '/' ~ ansible_hostname ~ '/' ~ item) | from_yaml | json_query(item ~ '.id')}}"
- name: "Render deployment file for {{ item }}"
template:
src: ../templates/heat-config.j2
copy:
content: "[ {{ lookup('file', role_name ~ '/' ~ ansible_hostname ~ '/' ~ item) | from_yaml | json_query(item) }} ]"
dest: "/var/lib/heat-config/tripleo-config-download/{{ item }}"
become: true
vars:
deployment: "{{ vars[item] | to_json }}"
- name: "Force remove deployed file for {{ item }}"
file:
path: /var/lib/heat-config/deployed/{{ vars[item].id }}.json
path: /var/lib/heat-config/deployed/{{ deployment_uuid }}.json
state: absent
become: true
when: force | bool
@ -16,7 +18,7 @@
- name: "Run deployment {{ item }}"
shell: |
/usr/libexec/os-refresh-config/configure.d/55-heat-config
exit $(jq .deploy_status_code /var/lib/heat-config/deployed/{{ vars[item].id }}.notify.json)
exit $(jq .deploy_status_code /var/lib/heat-config/deployed/{{ deployment_uuid }}.notify.json)
become: true
environment:
HEAT_SHELL_CONFIG: /var/lib/heat-config/tripleo-config-download/{{ item }}

74
tripleo_common/tests/utils/data/overcloud-controller-0
View File

@ -1,74 +0,0 @@
deploy_server_id: 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
ControllerHostEntryDeployment:
config: |
#!/bin/bash
echo "A script"
creation_time: "None"
deployment_name: ControllerHostEntryDeployment
group: script
id: 50fcb6ee-4ff2-4318-b734-d3a7b45a8d6d
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
name: None
options: None
outputs:
NetworkDeployment:
config: |
#!/bin/bash
echo "Another script"
creation_time: "None"
deployment_name: NetworkDeployment
group: script
id: 613b3009-7708-4b06-8c2a-56f8cd379aff
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
name: None
options: None
outputs:
MyPostConfig:
config: |
#!/bin/bash
echo "A PostConfig script"
creation_time: "None"
deployment_name: MyPostConfig
group: script
id: ac63372f-bddd-45a0-aca9-6f0950d581e2
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
name: None
options: None
outputs:
MyExtraConfigPost:
config: |
#!/bin/bash
echo "An ExtraConfigPost script"
creation_time: "None"
deployment_name: MyExtraConfigPost
group: script
id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
name: None
options: None
outputs:

19
tripleo_common/tests/utils/data/overcloud-controller-0/ControllerHostEntryDeployment Normal file
View File

@ -0,0 +1,19 @@
deploy_server_id: 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
ControllerHostEntryDeployment:
config: |
#!/bin/bash
echo "A script"
creation_time: "None"
deployment_name: ControllerHostEntryDeployment
group: script
id: 50fcb6ee-4ff2-4318-b734-d3a7b45a8d6d
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
name: None
options: None
outputs:

20
tripleo_common/tests/utils/data/overcloud-controller-0/MyExtraConfigPost Normal file
View File

@ -0,0 +1,20 @@
deploy_server_id: 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
MyExtraConfigPost:
config: |
#!/bin/bash
echo "An ExtraConfigPost script"
creation_time: "None"
deployment_name: MyExtraConfigPost
group: script
id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
name: None
options: None
outputs:

19
tripleo_common/tests/utils/data/overcloud-controller-0/MyPostConfig Normal file
View File

@ -0,0 +1,19 @@
deploy_server_id: 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
MyPostConfig:
config: |
#!/bin/bash
echo "A PostConfig script"
creation_time: "None"
deployment_name: MyPostConfig
group: script
id: ac63372f-bddd-45a0-aca9-6f0950d581e2
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
name: None
options: None
outputs:

19
tripleo_common/tests/utils/data/overcloud-controller-0/NetworkDeployment Normal file
View File

@ -0,0 +1,19 @@
deploy_server_id: 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
NetworkDeployment:
config: |
#!/bin/bash
echo "Another script"
creation_time: "None"
deployment_name: NetworkDeployment
group: script
id: 613b3009-7708-4b06-8c2a-56f8cd379aff
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
00b3a5e1-5e8e-4b55-878b-2fa2271f15ad
name: None
options: None
outputs:

54
tripleo_common/tests/utils/data/overcloud-novacompute-0
View File

@ -1,54 +0,0 @@
deploy_server_id: a7db3010-a51f-4ae0-a791-2364d629d20d
ComputeHostEntryDeployment:
config:
{"hiera_key1": "hiera_value1", "hiera_key2": "hiera_value2"}
creation_time: "None"
deployment_name: ComputeHostEntryDeployment
group: hiera
id: 26db1678-fe18-4acb-86db-19885fccae4c
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
a7db3010-a51f-4ae0-a791-2364d629d20d
name: None
options: None
outputs:
NetworkDeployment:
config:
{"hiera_key3": "hiera_value3", "hiera_key4": {"hiera_key4_subkey1": "hiera_key4_subvalue1", "hiera_key4_subkey0": "hiera_key4_subvalue0"}}
creation_time: "None"
deployment_name: NetworkDeployment
group: hiera
id: d21aefd1-9d94-44c0-b73a-16c8d4bc605c
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
a7db3010-a51f-4ae0-a791-2364d629d20d
name: None
options: None
outputs:
MyExtraConfigPost:
config: |
#!/bin/bash
echo "An ExtraConfigPost script"
creation_time: "None"
deployment_name: MyExtraConfigPost
group: script
id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
a7db3010-a51f-4ae0-a791-2364d629d20d
name: None
options: None
outputs:

18
tripleo_common/tests/utils/data/overcloud-novacompute-0/ComputeHostEntryDeployment Normal file
View File

@ -0,0 +1,18 @@
deploy_server_id: a7db3010-a51f-4ae0-a791-2364d629d20d
ComputeHostEntryDeployment:
config:
{"hiera_key1": "hiera_value1", "hiera_key2": "hiera_value2"}
creation_time: "None"
deployment_name: ComputeHostEntryDeployment
group: hiera
id: 26db1678-fe18-4acb-86db-19885fccae4c
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
a7db3010-a51f-4ae0-a791-2364d629d20d
name: None
options: None
outputs:

19
tripleo_common/tests/utils/data/overcloud-novacompute-0/MyExtraConfigPost Normal file
View File

@ -0,0 +1,19 @@
deploy_server_id: a7db3010-a51f-4ae0-a791-2364d629d20d
MyExtraConfigPost:
config: |
#!/bin/bash
echo "An ExtraConfigPost script"
creation_time: "None"
deployment_name: MyExtraConfigPost
group: script
id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
a7db3010-a51f-4ae0-a791-2364d629d20d
name: None
options: None
outputs:

18
tripleo_common/tests/utils/data/overcloud-novacompute-0/NetworkDeployment Normal file
View File

@ -0,0 +1,18 @@
deploy_server_id: a7db3010-a51f-4ae0-a791-2364d629d20d
NetworkDeployment:
config:
{"hiera_key3": "hiera_value3", "hiera_key4": {"hiera_key4_subkey1": "hiera_key4_subvalue1", "hiera_key4_subkey0": "hiera_key4_subvalue0"}}
creation_time: "None"
deployment_name: NetworkDeployment
group: hiera
id: d21aefd1-9d94-44c0-b73a-16c8d4bc605c
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
a7db3010-a51f-4ae0-a791-2364d629d20d
name: None
options: None
outputs:

54
tripleo_common/tests/utils/data/overcloud-novacompute-1
View File

@ -1,54 +0,0 @@
deploy_server_id: 8b07cd31-3083-4b88-a433-955f72039e2c
ComputeHostEntryDeployment:
config:
{"hiera_key1": "hiera_value1", "hiera_key2": "hiera_value2"}
creation_time: "None"
deployment_name: ComputeHostEntryDeployment
group: hiera
id: 26db1678-fe18-4acb-86db-19885fccae4c
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
8b07cd31-3083-4b88-a433-955f72039e2c
name: None
options: None
outputs:
NetworkDeployment:
config:
{"hiera_key3": "hiera_value3", "hiera_key4": {"hiera_key4_subkey1": "hiera_key4_subvalue1", "hiera_key4_subkey0": "hiera_key4_subvalue0"}}
creation_time: "None"
deployment_name: NetworkDeployment
group: hiera
id: d21aefd1-9d94-44c0-b73a-16c8d4bc605c
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
8b07cd31-3083-4b88-a433-955f72039e2c
name: None
options: None
outputs:
MyExtraConfigPost:
config: |
#!/bin/bash
echo "An ExtraConfigPost script"
creation_time: "None"
deployment_name: MyExtraConfigPost
group: script
id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
8b07cd31-3083-4b88-a433-955f72039e2c
name: None
options: None
outputs:

18
tripleo_common/tests/utils/data/overcloud-novacompute-1/ComputeHostEntryDeployment Normal file
View File

@ -0,0 +1,18 @@
deploy_server_id: 8b07cd31-3083-4b88-a433-955f72039e2c
ComputeHostEntryDeployment:
config:
{"hiera_key1": "hiera_value1", "hiera_key2": "hiera_value2"}
creation_time: "None"
deployment_name: ComputeHostEntryDeployment
group: hiera
id: 26db1678-fe18-4acb-86db-19885fccae4c
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
8b07cd31-3083-4b88-a433-955f72039e2c
name: None
options: None
outputs:

20
tripleo_common/tests/utils/data/overcloud-novacompute-1/MyExtraConfigPost Normal file
View File

@ -0,0 +1,20 @@
deploy_server_id: 8b07cd31-3083-4b88-a433-955f72039e2c
MyExtraConfigPost:
config: |
#!/bin/bash
echo "An ExtraConfigPost script"
creation_time: "None"
deployment_name: MyExtraConfigPost
group: script
id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
8b07cd31-3083-4b88-a433-955f72039e2c
name: None
options: None
outputs:

18
tripleo_common/tests/utils/data/overcloud-novacompute-1/NetworkDeployment Normal file
View File

@ -0,0 +1,18 @@
deploy_server_id: 8b07cd31-3083-4b88-a433-955f72039e2c
NetworkDeployment:
config:
{"hiera_key3": "hiera_value3", "hiera_key4": {"hiera_key4_subkey1": "hiera_key4_subvalue1", "hiera_key4_subkey0": "hiera_key4_subvalue0"}}
creation_time: "None"
deployment_name: NetworkDeployment
group: hiera
id: d21aefd1-9d94-44c0-b73a-16c8d4bc605c
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
8b07cd31-3083-4b88-a433-955f72039e2c
name: None
options: None
outputs:

74
tripleo_common/tests/utils/data/overcloud-novacompute-2
View File

@ -1,74 +0,0 @@
deploy_server_id: 169b46f8-1965-4d90-a7de-f36fb4a830fe
ComputeHostEntryDeployment:
config:
{"hiera_key1": "hiera_value1", "hiera_key2": "hiera_value2"}
creation_time: "None"
deployment_name: ComputeHostEntryDeployment
group: hiera
id: 26db1678-fe18-4acb-86db-19885fccae4c
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
169b46f8-1965-4d90-a7de-f36fb4a830fe
name: None
options: None
outputs:
NetworkDeployment:
config:
{"hiera_key3": "hiera_value3", "hiera_key4": {"hiera_key4_subkey1": "hiera_key4_subvalue1", "hiera_key4_subkey0": "hiera_key4_subvalue0"}}
creation_time: "None"
deployment_name: NetworkDeployment
group: hiera
id: d21aefd1-9d94-44c0-b73a-16c8d4bc605c
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
169b46f8-1965-4d90-a7de-f36fb4a830fe
name: None
options: None
outputs:
MyExtraConfigPost:
config: |
#!/bin/bash
echo "An ExtraConfigPost script"
creation_time: "None"
deployment_name: MyExtraConfigPost
group: script
id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
169b46f8-1965-4d90-a7de-f36fb4a830fe
name: None
options: None
outputs:
AnsibleDeployment:
id: 391263ba-3d06-43b6-a47e-481808aaff20
creation_time: "None"
deployment_name: AnsibleDeployment
name: None
options: None
group: ansible
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
169b46f8-1965-4d90-a7de-f36fb4a830fe
outputs:
config: |
tasks:
- name: An Ansible task
copy:
content: "{{ some_hostvar | to_json }}"
dest: /some/path

22
tripleo_common/tests/utils/data/overcloud-novacompute-2/AnsibleDeployment Normal file
View File

@ -0,0 +1,22 @@
deploy_server_id: 169b46f8-1965-4d90-a7de-f36fb4a830fe
AnsibleDeployment:
id: 391263ba-3d06-43b6-a47e-481808aaff20
creation_time: "None"
deployment_name: AnsibleDeployment
name: None
options: None
group: ansible
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
169b46f8-1965-4d90-a7de-f36fb4a830fe
outputs:
config: |
tasks:
- name: An Ansible task
copy:
content: "{{ some_hostvar | to_json }}"
dest: /some/path

18
tripleo_common/tests/utils/data/overcloud-novacompute-2/ComputeHostEntryDeployment Normal file
View File

@ -0,0 +1,18 @@
deploy_server_id: 169b46f8-1965-4d90-a7de-f36fb4a830fe
ComputeHostEntryDeployment:
config:
{"hiera_key1": "hiera_value1", "hiera_key2": "hiera_value2"}
creation_time: "None"
deployment_name: ComputeHostEntryDeployment
group: hiera
id: 26db1678-fe18-4acb-86db-19885fccae4c
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
169b46f8-1965-4d90-a7de-f36fb4a830fe
name: None
options: None
outputs:

19
tripleo_common/tests/utils/data/overcloud-novacompute-2/MyExtraConfigPost Normal file
View File

@ -0,0 +1,19 @@
deploy_server_id: 169b46f8-1965-4d90-a7de-f36fb4a830fe
MyExtraConfigPost:
config: |
#!/bin/bash
echo "An ExtraConfigPost script"
creation_time: "None"
deployment_name: MyExtraConfigPost
group: script
id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
169b46f8-1965-4d90-a7de-f36fb4a830fe
name: None
options: None
outputs:

18
tripleo_common/tests/utils/data/overcloud-novacompute-2/NetworkDeployment Normal file
View File

@ -0,0 +1,18 @@
deploy_server_id: 169b46f8-1965-4d90-a7de-f36fb4a830fe
NetworkDeployment:
config:
{"hiera_key3": "hiera_value3", "hiera_key4": {"hiera_key4_subkey1": "hiera_key4_subvalue1", "hiera_key4_subkey0": "hiera_key4_subvalue0"}}
creation_time: "None"
deployment_name: NetworkDeployment
group: hiera
id: d21aefd1-9d94-44c0-b73a-16c8d4bc605c
inputs:
- name: deploy_server_id
description: None
type: None
value: |-
169b46f8-1965-4d90-a7de-f36fb4a830fe
name: None
options: None
outputs:

56
tripleo_common/tests/utils/test_config.py
View File

@ -277,11 +277,7 @@ class TestConfig(base.TestCase):
self.tmp_dir = self.useFixture(fixtures.TempDir()).path
tmp_path = self.config.download_config(stack, self.tmp_dir)
for f in ['overcloud-controller-0',
'overcloud-novacompute-0',
'overcloud-novacompute-1',
'overcloud-novacompute-2',
'Controller',
for f in ['Controller',
'Compute', ]:
self.assertEqual(
@ -289,6 +285,56 @@ class TestConfig(base.TestCase):
open(os.path.join(tmp_path, 'group_vars', f)).read()),
self._get_yaml_file(f))
for d in ['ControllerHostEntryDeployment',
'NetworkDeployment',
'MyExtraConfigPost',
'MyPostConfig']:
self.assertEqual(
yaml.safe_load(
open(os.path.join(tmp_path, 'Controller',
'overcloud-controller-0',
d)).read()),
self._get_yaml_file(os.path.join(
'overcloud-controller-0',
d)))
for d in ['ComputeHostEntryDeployment',
'NetworkDeployment',
'MyExtraConfigPost']:
self.assertEqual(
yaml.safe_load(
open(os.path.join(tmp_path, 'Compute',
'overcloud-novacompute-0',
d)).read()),
self._get_yaml_file(os.path.join(
'overcloud-novacompute-0',
d)))
for d in ['ComputeHostEntryDeployment',
'NetworkDeployment',
'MyExtraConfigPost']:
self.assertEqual(
yaml.safe_load(
open(os.path.join(tmp_path, 'Compute',
'overcloud-novacompute-1',
d)).read()),
self._get_yaml_file(os.path.join(
'overcloud-novacompute-1',
d)))
for d in ['ComputeHostEntryDeployment',
'NetworkDeployment',
'MyExtraConfigPost',
'AnsibleDeployment']:
self.assertEqual(
yaml.safe_load(
open(os.path.join(tmp_path, 'Compute',
'overcloud-novacompute-2',
d)).read()),
self._get_yaml_file(os.path.join(
'overcloud-novacompute-2',
d)))
@patch('tripleo_common.utils.config.Config.get_config_dict')
@patch('tripleo_common.utils.config.Config.get_deployment_data')
def test_config_download_os_apply_config(

27
tripleo_common/utils/config.py
View File

@ -137,7 +137,7 @@ class Config(object):
def _mkdir(self, dirname):
if not os.path.exists(dirname):
try:
os.mkdir(dirname, 0o700)
os.makedirs(dirname, 0o700)
except OSError as e:
message = 'Failed to create: %s, error: %s' % (dirname,
str(e))
@ -211,6 +211,9 @@ class Config(object):
# that role. The deployment names are futher separated in their own
# dict with keys of pre_deployment/post_deployment.
role_deployment_names = {}
# server_roles is a dict of server name to server role for easier
# lookup
server_roles = {}
for deployment in deployments_data:
server_id = deployment.attributes['value']['server']
@ -241,6 +244,8 @@ class Config(object):
role_post_deployments = role_deployments.setdefault(
'post_deployments', [])
server_roles[server_names[server_id]] = role
# special handling of deployments that are run post the deploy
# steps. We have to look these up based on the
# physical_resource_id, but these names should be consistent since
@ -264,11 +269,16 @@ class Config(object):
self._mkdir(group_vars_dir)
for server, deployments in server_deployments.items():
group_var_server_path = os.path.join(group_vars_dir, server)
group_var_server_template = env.get_template('group_var_server.j2')
deployment_template = env.get_template('deployment.j2')
for d in deployments:
server_deployment_dir = os.path.join(
tmp_path, server_roles[server], server)
self._mkdir(server_deployment_dir)
deployment_path = os.path.join(
server_deployment_dir, d['deployment_name'])
# See if the config can be loaded as a JSON data structure
# In some cases, it may already be JSON (hiera), or it may just
# be a string (script). In those cases, just use the value
@ -280,7 +290,8 @@ class Config(object):
# If the value is not a string already, pretty print it as a
# string so it's rendered in a readable format.
if not isinstance(data, six.text_type):
if not (isinstance(data, six.text_type) or
isinstance(data, six.string_types)):
data = json.dumps(data, indent=2)
d['config'] = data
@ -298,10 +309,10 @@ class Config(object):
"config-download." % d['deployment_name'])
warnings.warn(message, DeprecationWarning)
with open(group_var_server_path, 'w') as f:
f.write(group_var_server_template.render(
deployments=deployments,
server_id=server_ids[server]))
with open(deployment_path, 'w') as f:
f.write(deployment_template.render(
deployment=d,
server_id=server_ids[server]))
for role, deployments in role_deployment_names.items():
group_var_role_path = os.path.join(group_vars_dir, role)