Cleanup files and templates using smart sources

The files and templates we carry are almost always in a state of
maintenance. The upstream services are maintaining these files and
there's really no reason we need to carry duplicate copies of them. This
change removes all of the files we expect to get from the upstream
service. while the focus of this change is to remove configuration file
maintenance burdens it also allows the role to execute faster.

  * Source installs have the configuration files within the venv at
    "<<VENV_PATH>>/etc/<<SERVICE_NAME>>". The role will now link the
    default configuration path to this directory. When the service is
    upgraded the link will move to the new venv path.
  * Distro installs package all of the required configuration files.

To maintain our current capabilities to override configuration the
role will fetch files from the disk whenever an override is provided and
then push the fetched file back to the target using `config_template`.

Change-Id: Ib26b25c1df483c2e31bccc91874a83292be6a3aa
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
This commit is contained in:
Kevin Carter 2018-08-05 00:14:48 -05:00
parent a839cc1713
commit 36a5e628e0
No known key found for this signature in database
GPG Key ID: 9443251A787B9FB3
8 changed files with 133 additions and 363 deletions

View File

@ -296,5 +296,6 @@ heat_role_project_group: heat_all
heat_heat_conf_overrides: {}
heat_api_paste_ini_overrides: {}
heat_default_yaml_overrides: {}
heat_aws_cloudwatch_alarm_yaml_overrides: {}
heat_aws_rds_dbinstance_yaml_overrides: {}
heat_policy_overrides: {}

View File

@ -15,30 +15,13 @@
- name: Drop heat Config(s)
config_template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
src: "heat.conf.j2"
dest: "/etc/heat/heat.conf"
owner: "root"
group: "{{ heat_system_group_name }}"
mode: "0640"
config_overrides: "{{ item.config_overrides }}"
config_type: "{{ item.config_type }}"
with_items:
- src: "heat.conf.j2"
dest: "/etc/heat/heat.conf"
config_overrides: "{{ heat_heat_conf_overrides }}"
config_type: "ini"
- src: "api-paste.ini.j2"
dest: "/etc/heat/api-paste.ini"
config_overrides: "{{ heat_api_paste_ini_overrides }}"
config_type: "ini"
- src: "environment.d/default.yaml.j2"
dest: "/etc/heat/environment.d/default.yaml"
config_overrides: "{{ heat_default_yaml_overrides }}"
config_type: "yaml"
- src: "templates/AWS_RDS_DBInstance.yaml.j2"
dest: "/etc/heat/templates/AWS_RDS_DBInstance.yaml"
config_overrides: "{{ heat_aws_rds_dbinstance_yaml_overrides }}"
config_type: "yaml"
config_overrides: "{{ heat_heat_conf_overrides }}"
config_type: "ini"
notify:
- Restart heat services
@ -48,3 +31,50 @@
dest: "/etc/heat/policy.json"
when:
- heat_policy_overrides != {}
# NOTE(cloudnull): This is using "cp" instead of copy with a remote_source
# because we only want to copy the original files once. and we
# don't want to need multiple tasks.
- name: Preserve original configuration file(s)
command: "cp {{ item.target_f }} {{ item.target_f }}.original"
args:
creates: "{{ item.target_f }}.original"
when:
- item.condition | bool
with_items: "{{ heat_core_files }}"
- name: Fetch override files
fetch:
src: "{{ item.target_f }}"
dest: "{{ item.tmp_f }}"
flat: yes
changed_when: false
run_once: true
when:
- item.condition | bool
with_items: "{{ heat_core_files }}"
- name: Copy common config
config_template:
src: "{{ item.tmp_f }}"
dest: "{{ item.target_f }}"
owner: "root"
group: "{{ item.group | default(heat_system_group_name) }}"
mode: "0640"
config_overrides: "{{ item.config_overrides }}"
config_type: "{{ item.config_type }}"
when:
- item.condition | bool
with_items: "{{ heat_core_files }}"
notify:
- Restart heat services
- name: Cleanup fetched temp files
file:
path: "{{ item.tmp_f }}"
state: absent
changed_when: false
delegate_to: localhost
when:
- item.condition | bool
with_items: "{{ heat_core_files }}"

View File

@ -29,21 +29,61 @@
createhome: "yes"
home: "{{ heat_system_home_folder }}"
# NOTE(cloudnull): During an upgrade the local directory may exist on a source
# install. If the directory does exist it will need to be
# removed. This is required on source installs because the
# config directory is a link.
- name: Source config block
block:
- name: Stat config directory
stat:
path: "/etc/heat"
register: heat_conf_dir_stat
- name: Remove the config directory
file:
path: "/etc/heat"
state: absent
when:
- heat_conf_dir_stat.stat.isdir is defined and
heat_conf_dir_stat.stat.isdir
when:
- heat_install_method == 'source'
- name: Create heat dir
file:
path: "{{ item.path }}"
state: directory
path: "{{ item.path | default(omit) }}"
src: "{{ item.src | default(omit) }}"
dest: "{{ item.dest | default(omit) }}"
state: "{{ item.state | default('directory') }}"
owner: "{{ item.owner|default(heat_system_user_name) }}"
group: "{{ item.group|default(heat_system_group_name) }}"
mode: "{{ item.mode|default('0755') }}"
mode: "{{ item.mode | default(omit) }}"
force: "{{ item.force | default(omit) }}"
when:
- (item.condition | default(true)) | bool
with_items:
- { path: "/openstack", owner: "root", group: "root" }
- { path: "/openstack/venvs", owner: "root", group: "root" }
- { path: "/etc/heat", mode: "0750" }
- { path: "/etc/heat/environment.d" }
- { path: "/etc/heat/templates" }
- { path: "/var/cache/heat", mode: "0700" }
- { path: "{{ heat_system_home_folder }}" }
- path: "/openstack"
owner: "root"
group: "root"
- path: "/openstack/venvs"
owner: "root"
group: "root"
- path: "{{ (heat_install_method == 'distro') | ternary('/etc/heat', (heat_bin | dirname) + '/etc/heat') }}"
mode: "0755"
# NOTE(cloudnull): The "src" path is relative. This ensures all files remain
# within the host/container confines when connecting to
# them using the connection plugin or the root filesystem.
- dest: "/etc/heat"
src: "{{ heat_bin | dirname | regex_replace('^/', '../') }}/etc/heat"
state: link
force: true
condition: "{{ heat_install_method == 'source' }}"
- path: "/etc/heat/environment.d"
- path: "/etc/heat/templates"
- path: "/var/cache/heat"
mode: "0700"
- path: "{{ heat_system_home_folder }}"
- name: Create heat plugin dirs
file:

View File

@ -1,102 +0,0 @@
# heat-api pipeline
[pipeline:heat-api]
pipeline = cors request_id faultwrap http_proxy_to_wsgi versionnegotiation osprofiler authurl authtoken context apiv1app
# heat-api pipeline for standalone heat
# ie. uses alternative auth backend that authenticates users against keystone
# using username and password instead of validating token (which requires
# an admin/service token).
# To enable, in heat.conf:
# [paste_deploy]
# flavor = standalone
#
[pipeline:heat-api-standalone]
pipeline = cors request_id faultwrap http_proxy_to_wsgi versionnegotiation authurl authpassword context apiv1app
# heat-api pipeline for custom cloud backends
# i.e. in heat.conf:
# [paste_deploy]
# flavor = custombackend
#
[pipeline:heat-api-custombackend]
pipeline = cors request_id faultwrap versionnegotiation context custombackendauth apiv1app
# To enable, in heat.conf:
# [paste_deploy]
# flavor = noauth
#
[pipeline:heat-api-noauth]
pipeline = cors request_id faultwrap http_proxy_to_wsgi versionnegotiation noauth context apiv1app
# heat-api-cfn pipeline
[pipeline:heat-api-cfn]
pipeline = cors http_proxy_to_wsgi cfnversionnegotiation osprofiler ec2authtoken authtoken context apicfnv1app
# heat-api-cfn pipeline for standalone heat
# relies exclusively on authenticating with ec2 signed requests
[pipeline:heat-api-cfn-standalone]
pipeline = cors http_proxy_to_wsgi cfnversionnegotiation ec2authtoken context apicfnv1app
[app:apiv1app]
paste.app_factory = heat.common.wsgi:app_factory
heat.app_factory = heat.api.openstack.v1:API
[app:apicfnv1app]
paste.app_factory = heat.common.wsgi:app_factory
heat.app_factory = heat.api.cfn.v1:API
[filter:versionnegotiation]
paste.filter_factory = heat.common.wsgi:filter_factory
heat.filter_factory = heat.api.openstack:version_negotiation_filter
[filter:cors]
paste.filter_factory = oslo_middleware.cors:filter_factory
oslo_config_project = heat
[filter:faultwrap]
paste.filter_factory = heat.common.wsgi:filter_factory
heat.filter_factory = heat.api.openstack:faultwrap_filter
[filter:cfnversionnegotiation]
paste.filter_factory = heat.common.wsgi:filter_factory
heat.filter_factory = heat.api.cfn:version_negotiation_filter
[filter:cwversionnegotiation]
paste.filter_factory = heat.common.wsgi:filter_factory
[filter:context]
paste.filter_factory = heat.common.context:ContextMiddleware_filter_factory
[filter:ec2authtoken]
paste.filter_factory = heat.api.aws.ec2token:EC2Token_filter_factory
[filter:http_proxy_to_wsgi]
paste.filter_factory = oslo_middleware:HTTPProxyToWSGI.factory
# Middleware to set auth_url header appropriately
[filter:authurl]
paste.filter_factory = heat.common.auth_url:filter_factory
# Auth middleware that validates token against keystone
[filter:authtoken]
paste.filter_factory = keystonemiddleware.auth_token:filter_factory
# Auth middleware that validates username/password against keystone
[filter:authpassword]
paste.filter_factory = heat.common.auth_password:filter_factory
# Auth middleware that validates against custom backend
[filter:custombackendauth]
paste.filter_factory = heat.common.custom_backend_auth:filter_factory
# Auth middleware that accepts any auth
[filter:noauth]
paste.filter_factory = heat.common.noauth:filter_factory
# Middleware to set x-openstack-request-id in http response header
[filter:request_id]
paste.filter_factory = oslo_middleware.request_id:RequestId.factory
[filter:osprofiler]
paste.filter_factory = osprofiler.web:WsgiMiddleware.factory

View File

@ -1,14 +0,0 @@
resource_registry:
# allow older templates with Quantum in them.
"OS::Quantum*": "OS::Neutron*"
# Choose your implementation of AWS::CloudWatch::Alarm
"AWS::CloudWatch::Alarm": "file:///etc/heat/templates/AWS_CloudWatch_Alarm.yaml"
#"AWS::CloudWatch::Alarm": "OS::Heat::CWLiteAlarm"
"OS::Metering::Alarm": "OS::Aodh::Alarm"
"AWS::RDS::DBInstance": "file:///etc/heat/templates/AWS_RDS_DBInstance.yaml"
"OS::Ceilometer::Alarm": "OS::Aodh::Alarm"
"OS::Ceilometer::GnocchiResourcesAlarm": "OS::Aodh::GnocchiResourcesAlarm"
"OS::Ceilometer::GnocchiAggregationByMetricsAlarm": "OS::Aodh::GnocchiAggregationByMetricsAlarm"
"OS::Ceilometer::GnocchiAggregationByResourcesAlarm": "OS::Aodh::GnocchiAggregationByResourcesAlarm"
"OS::Ceilometer::CombinationAlarm": "OS::Aodh::CombinationAlarm"

View File

@ -1,87 +0,0 @@
HeatTemplateFormatVersion: '2012-12-12'
Description: AWS::CloudWatch::Alarm using Aodh.
Parameters:
AlarmDescription:
Type: String
Default: An alarm
EvaluationPeriods:
Type: String
MetricName:
Type: String
Namespace:
Type: String
Default: system/linux
Period:
Type: String
ComparisonOperator:
Type: String
AllowedValues: [GreaterThanOrEqualToThreshold, GreaterThanThreshold,
LessThanThreshold, LessThanOrEqualToThreshold]
Statistic:
Type: String
AllowedValues: [SampleCount, Average, Sum, Minimum, Maximum]
Threshold:
Type: String
Units:
Type: String
AllowedValues: [Seconds, Microseconds, Milliseconds, Bytes, Kilobytes,
Megabytes, Gigabytes, Terabytes, Bits, Kilobits, Megabits,
Gigabits, Terabits, Percent, Count, Bytes/Second,
Kilobytes/Second, Megabytes/Second, Gigabytes/Second,
Terabytes/Second, Bits/Second, Kilobits/Second,
Megabits/Second, Gigabits/Second, Terabits/Second,
Count/Second, None]
Default: None
AlarmActions:
Type: CommaDelimitedList
Default: ''
OKActions:
Type: CommaDelimitedList
Default: ''
InsufficientDataActions:
Type: CommaDelimitedList
Default: ''
Dimensions:
Type: CommaDelimitedList
Default: ''
Mappings:
ComparisonOperatorMap:
LessThanOrEqualToThreshold: {Aodh: le}
LessThanThreshold: {Aodh: lt}
GreaterThanThreshold: {Aodh: gt}
GreaterThanOrEqualToThreshold: {Aodh: ge}
StatisticMap:
SampleCount: {Aodh: count}
Average: {Aodh: avg}
Sum: {Aodh: sum}
Minimum: {Aodh: min}
Maximum: {Aodh: max}
Resources:
__alarm__:
Type: OS::Aodh::Alarm
Properties:
description:
Ref: AlarmDescription
meter_name:
Ref: MetricName
period:
Ref: Period
evaluation_periods:
Ref: EvaluationPeriods
repeat_actions: true
threshold:
Ref: Threshold
alarm_actions:
Ref: AlarmActions
ok_actions:
Ref: OKActions
insufficient_data_actions:
Ref: InsufficientDataActions
statistic:
"Fn::FindInMap": [StatisticMap, {Ref: Statistic}, Aodh]
comparison_operator:
"Fn::FindInMap": [ComparisonOperatorMap, {Ref: ComparisonOperator}, Aodh]
matching_metadata:
"Fn::MemberListToMap": [Name, Value, {Ref: Dimensions}]

View File

@ -1,129 +0,0 @@
HeatTemplateFormatVersion: '2012-12-12'
Description: 'Builtin AWS::RDS::DBInstance'
Parameters:
AllocatedStorage:
Type: String
DBInstanceClass:
Type: String
DBName:
Type: String
DBSecurityGroups:
Type: CommaDelimitedList
Default: ''
Engine:
Type: String
AllowedValues: ['MySQL']
MasterUsername:
Type: String
MasterUserPassword:
Type: String
Port:
Type: String
Default: '3306'
KeyName:
Type: String
Default: ''
Mappings:
DBInstanceToInstance:
db.m1.small: {Instance: m1.small}
db.m1.large: {Instance: m1.large}
db.m1.xlarge: {Instance: m1.xlarge}
db.m2.xlarge: {Instance: m2.xlarge}
db.m2.2xlarge: {Instance: m2.2xlarge}
db.m2.4xlarge: {Instance: m2.4xlarge}
Resources:
ServerSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: 'Enable SSH access'
SecurityGroupIngress:
- IpProtocol: icmp
FromPort: '-1'
ToPort: '-1'
CidrIp: '0.0.0.0/0'
- IpProtocol: tcp
FromPort: '22'
ToPort : '22'
CidrIp : '0.0.0.0/0'
- IpProtocol: tcp
FromPort: {Ref: Port}
ToPort : {Ref: Port}
CidrIp : '0.0.0.0/0'
DatabaseInstance:
Type: AWS::EC2::Instance
Metadata:
AWS::CloudFormation::Init:
config:
files:
/tmp/db_setup.sql:
content:
'Fn::Replace':
- DBName: {Ref: DBName}
MasterUserPassword: {Ref: MasterUserPassword}
MasterUsername: {Ref: MasterUsername}
- |
CREATE DATABASE DBName;
GRANT ALL PRIVILEGES ON DBName.* TO "MasterUsername"@"%"
IDENTIFIED BY "MasterUserPassword";
FLUSH PRIVILEGES;
EXIT
mode: '000644'
owner: root
group: root
packages:
yum:
mariadb: []
mariadb-server: []
services:
systemd:
mysqld:
enabled: true
ensureRunning: true
Properties:
ImageId: F19-x86_64-cfntools
InstanceType: {'Fn::FindInMap': [DBInstanceToInstance,
{Ref: DBInstanceClass}, Instance]}
KeyName: {Ref: KeyName}
SecurityGroups: [{"Ref" : "ServerSecurityGroup"}]
UserData:
Fn::Base64:
Fn::Replace:
- 'AWS::StackName': {Ref: 'AWS::StackName'}
'AWS::Region': {Ref: 'AWS::Region'}
MasterUserPassword: {Ref: MasterUserPassword}
WaitHandle: {Ref: WaitHandle}
- |
#!/bin/bash -v
#
iptables -F
# Helper function
function error_exit
{
/opt/aws/bin/cfn-signal -e 1 -r \"$1\" 'WaitHandle'
exit 1
}
/opt/aws/bin/cfn-init -s AWS::StackName -r DatabaseInstance --region AWS::Region || error_exit 'Failed to run cfn-init'
# Setup MySQL root password and create a user
mysqladmin -u root password 'MasterUserPassword'
mysql -u root --password='MasterUserPassword' < /tmp/db_setup.sql || error_exit 'Failed to setup mysql'
# Database setup completed, signal success
/opt/aws/bin/cfn-signal -e 0 -r "MySQL server setup complete" 'WaitHandle'
WaitHandle:
Type: AWS::CloudFormation::WaitConditionHandle
WaitCondition:
Type: AWS::CloudFormation::WaitCondition
DependsOn: DatabaseInstance
Properties:
Handle: {Ref: WaitHandle}
Timeout: "600"
Outputs:
Endpoint.Address:
Value: {'Fn::GetAtt': [DatabaseInstance, PublicIp]}
Endpoint.Port:
Value: {Ref: Port}

View File

@ -29,3 +29,34 @@ filtered_heat_services: |-
{% endif %}
{% endfor %}
{{ services | sort(attribute='start_order') }}
heat_core_files:
- tmp_f: "/tmp/api-paste.ini"
target_f: "{{ (heat_install_method == 'distro' and ansible_os_family == 'RedHat') | ternary('/usr/share/heat', '/etc/heat') }}/api-paste.ini"
config_overrides: "{{ heat_api_paste_ini_overrides }}"
config_type: "ini"
condition: "{{ not ((heat_install_method == 'distro') and ((ansible_os_family | lower) == 'redhat')) }}"
- tmp_f: "/tmp/default.yaml"
target_f: "/etc/heat/environment.d/default.yaml"
config_overrides: "{{ heat_default_yaml_overrides }}"
config_type: "yaml"
owner: "root"
group: "{{ heat_system_group_name }}"
mode: "0640"
condition: true
- tmp_f: "/tmp/AWS_CloudWatch_Alarm.yaml"
target_f: "/etc/heat/templates/AWS_CloudWatch_Alarm.yaml"
config_overrides: "{{ heat_aws_cloudwatch_alarm_yaml_overrides }}"
config_type: "yaml"
owner: "root"
group: "{{ heat_system_group_name }}"
mode: "0640"
condition: true
- tmp_f: "/tmp/AWS_RDS_DBInstance.yaml"
target_f: "/etc/heat/templates/AWS_RDS_DBInstance.yaml"
config_overrides: "{{ heat_aws_rds_dbinstance_yaml_overrides }}"
config_type: "yaml"
owner: "root"
group: "{{ heat_system_group_name }}"
mode: "0640"
condition: true