openstack
/
openstack-ansible-os_panko
Code Issues Proposed changes

Migrate panko to use uWSGI 

This patch moves panko-api from usage of apache with mod_wsgi
to uWSGI, which means unification across another roles and
reduced maintenance costs

During migration period tasks that ensures apache won't listen
on panko_service_port are present, but they are supposed to be removed
after train release.

Change-Id: Ic61c60e2db45ac5e51bb3a43d7928648d9ae5d19
This commit is contained in:
Dmitriy Rabotyagov 2019-08-02 15:42:24 +03:00
parent 46bb2417ea
commit 50f1a48c6c
14 changed files with 183 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
defaults/main.yml
View File

@ -57,10 +57,8 @@ panko_db_address: "{{ galera_address | default('127.0.0.1') }}"
panko_connection_string: "{{ panko_db_type }}://{{ panko_database_user }}:{{ panko_container_db_password }}@{{ panko_db_address }}/{{ panko_database_name }}?charset=utf8"
## Apache setup
panko_apache_log_level: info
panko_apache_servertokens: "Prod"
panko_apache_serversignature: "Off"
## uWSGI setup
panko_api_init_overrides: {}
panko_wsgi_threads: 10
panko_wsgi_processes_max: 16
panko_wsgi_processes: "{{ [[ansible_processor_vcpus|default(1), 1] | max * 2, panko_wsgi_processes_max] | min }}"
@ -72,6 +70,7 @@ panko_role_name: admin
panko_service_region: RegionOne
panko_service_endpoint_type: internalURL
panko_service_name: panko
panko_service_address: 0.0.0.0
panko_service_port: 8777
panko_service_proto: http
panko_service_type: event
@ -99,6 +98,15 @@ panko_event_time_to_live: -1
# reboot, yearly, annually, monthly, weekly, daily, hourly
panko_expirer_job_time: daily
panko_services:
panko-api:
group: "panko_api"
service_name: "panko-api"
service_enabled: true
config_overrides: "{{ panko_api_init_overrides }}"
execstarts: "{{ panko_bin }}/uwsgi --autoload --ini /etc/uwsgi/panko-api.ini"
execreloads: "{{ panko_bin }}/uwsgi --reload /var/run/panko-api/panko-api.pid"
# Common pip packages
panko_pip_packages:
- alembic>=0.7.2
@ -108,11 +116,11 @@ panko_pip_packages:
- osprofiler
- panko
- PyMySQL>=0.6.2
- python-ceilometerclient
- python-memcached
- sqlalchemy>=0.9.7
- sqlalchemy-utils
- systemd-python
- uwsgi
- warlock
@ -133,3 +141,4 @@ panko_role_project_group: panko_all
panko_policy_overrides: {}
panko_panko_conf_overrides: {}
panko_api_paste_ini_overrides: {}
panko_uwsgi_conf_overrides: {}

55
handlers/main.yml
View File

@ -13,6 +13,32 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Stop services
service:
name: "{{ item.value.service_name }}"
enabled: yes
state: "stopped"
daemon_reload: yes
with_dict: "{{ panko_services }}"
when:
- "item.value.group in group_names"
- item.value.service_enabled | bool
register: _stop
until: _stop is success
retries: 5
delay: 2
listen:
- "Restart panko services"
- "venv changed"
# Note (odyssey4me):
# The policy.json file is currently read continually by the services
# and is not only read on service start. We therefore cannot template
# directly to the file read by the service because the new policies
# may not be valid until the service restarts. This is particularly
# important during a major upgrade. We therefore only put the policy
# file in place after the service has been stopped.
#
- name: Copy new policy file into place
copy:
src: "/etc/panko/policy.json-{{ panko_venv_tag }}"
@ -22,18 +48,33 @@
mode: "0640"
remote_src: yes
listen:
- "Restart web server"
- "Restart panko services"
- "venv changed"
- name: Start services
service:
name: "{{ item.value.service_name }}"
enabled: yes
state: "started"
daemon_reload: yes
with_dict: "{{ panko_services }}"
when:
- "item.value.group in group_names"
- item.value.service_enabled | bool
register: _start
until: _start is success
retries: 5
delay: 2
listen:
- "Restart panko services"
- "venv changed"
# NOTE(noonedeadpunk): This task is created due to migration from apache to uwsgi
# which was introduced during train release. It can be dropped afterwards.
- name: Restart web server
service:
name: "{{ panko_system_service_name }}"
enabled: yes
state: "restarted"
daemon_reload: "{{ (ansible_service_mgr == 'systemd') | ternary('yes', omit) }}"
register: _restart
until: _restart is success
retries: 5
delay: 2
listen:
- "venv changed"
failed_when: false

12
releasenotes/notes/use_uwsgi-ec064b70a3b4e0c1.yaml Normal file
View File

@ -0,0 +1,12 @@
upgrade:
- |
Panko migrated from usage of Apache mod_wsgi or native daemon to uWSGI
daemon. This means, that `panko_apache_*` variables are not available and has no effect
anymore.
During upgrade process role will drop `panko_service_port` from apache
listeners (ports.conf) and panko virtualhost, which by default means
misconfigured apache service (since it won't have any listeners) unless
it's aio build and this apache server is in use by other role/service.
Apache server won't be dropped from panko_api hosts, so deployers
are encoureged to remove it manually.

37
tasks/main.yml
View File

@ -71,12 +71,43 @@
tags:
- panko-config
- import_tasks: panko_db_sync.yml
when: inventory_hostname == groups['panko_all'][0]
- name: Run the systemd service role
import_role:
name: systemd_service
vars:
systemd_user_name: "{{ panko_system_user_name }}"
systemd_group_name: "{{ panko_system_group_name }}"
systemd_tempd_prefix: openstack
systemd_slice_name: panko
systemd_lock_path: /var/lock/panko
systemd_CPUAccounting: true
systemd_BlockIOAccounting: true
systemd_MemoryAccounting: true
systemd_TasksAccounting: true
systemd_services: |-
{% set services = [] %}
{% for name, service in panko_services.items() %}
{% if (service['group'] in group_names) and
(('service_enabled' not in service) or
('service_enabled' in service and service['service_enabled'])) %}
{% set _ = service.update(
{
'service_key': name,
'enabled': 'yes',
'state': 'started',
}
)
%}
{% set _ = services.append(service) %}
{% endif %}
{% endfor %}
{{ services }}
tags:
- panko-config
- systemd-service
- import_tasks: panko_apache.yml
- import_tasks: panko_db_sync.yml
when: inventory_hostname == groups['panko_all'][0]
tags:
- panko-config

90
tasks/panko_apache.yml
View File

@ -1,90 +0,0 @@
---
# Copyright 2016, Comcast Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# NOTE(hwoarang) default openSUSE apache2 installation is missing
# some required modules so enable them here. This can possibly be
# extended to other distributions if necessary.
- name: Enable required apache2 modules (SUSE)
apache2_module:
name: "{{ item.name }}"
state: "{{ item.state }}"
with_items: "{{ panko_apache_modules }}"
when:
- ansible_pkg_mgr == 'zypper'
- panko_apache_modules is defined
notify:
- Restart web server
- name: Drop apache2 virtual host and ports file
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: "root"
group: "root"
with_items: "{{ panko_apache_config }}"
notify:
- Restart web server
- name: Disable default apache site
file:
path: "{{ item }}"
state: "absent"
with_items: "{{ panko_apache_default_sites }}"
notify:
- Restart web server
- name: Enabled panko vhost
file:
src: "{{ panko_apache_site_available }}"
dest: "{{ panko_apache_site_enabled }}"
state: "link"
when:
- panko_apache_site_available is defined
- panko_apache_site_enabled is defined
notify:
- Restart web server
- name: Ensure Apache ServerName
lineinfile:
dest: "{{ panko_apache_conf }}"
line: "ServerName {{ inventory_hostname }}"
notify:
- Restart web server
- name: Ensure Apache ServerTokens
lineinfile:
dest: "{{ panko_apache_security_conf }}"
regexp: '^ServerTokens'
line: "ServerTokens {{ panko_apache_servertokens }}"
notify:
- Restart web server
- name: Ensure Apache ServerSignature
lineinfile:
dest: "{{ panko_apache_security_conf }}"
regexp: '^ServerSignature'
line: "ServerSignature {{ panko_apache_serversignature }}"
notify:
- Restart web server
- name: remove Listen from Apache config
lineinfile:
dest: "{{ panko_apache_security_conf }}"
regexp: '^(Listen.*)'
backrefs: yes
line: '#\1'
notify:
- Restart web server

46
tasks/panko_post_install.yml
View File

@ -36,17 +36,43 @@
config_overrides: "{{ panko_policy_overrides }}"
config_type: "json"
notify:
- Restart web server
- Restart panko services
- name: Drop panko API WSGI Configs
template:
src: panko-api-wsgi.py.j2
dest: /var/www/cgi-bin/panko/panko-api
owner: "{{ panko_system_user_name }}"
group: "{{ panko_system_group_name }}"
mode: "0755"
- name: Configurng uWSGI
block:
- name: Ensure uWSGI directory exists
file:
path: "/etc/uwsgi/"
state: directory
mode: "0711"
- name: Apply uWSGI configuration
config_template:
src: "panko-uwsgi.ini.j2"
dest: "/etc/uwsgi/panko-api.ini"
mode: "0644"
config_overrides: "{{ panko_uwsgi_conf_overrides }}"
config_type: ini
notify:
- Restart panko services
when: "'panko_api' in group_names"
# NOTE(noonedeadpunk): This task is created due to migration from apache to uwsgi
# which was introduced during train release. It can be dropped afterwards.
- name: Drop Apache related configs
file:
path: "{{ item }}"
state: absent
with_items: "{{ panko_apache_config }}"
notify:
- Restart web server
- Restart web server
- name: Stop Apache listening on Panko port
lineinfile:
path: "{{ panko_apache_ports }}"
state: absent
line: "Listen {{ panko_service_port }}"
notify:
- Restart web server
- name: Create cron job for panko event expirer
cron:
@ -56,4 +82,4 @@
job: "{{ panko_bin }}/panko-expirer"
when:
- inventory_hostname == groups['panko_all'][0]
- panko_event_time_to_live > 0
- panko_event_time_to_live > 0

12
tasks/panko_pre_install.yml
View File

@ -42,14 +42,4 @@
- { path: "{{ panko_system_user_home }}" }
- { path: "{{ panko_system_user_home }}/.ssh", mode: "0700" }
- { path: "/var/cache/panko", mode: "0700" }
- name: Create Apache mod_wsgi dirs
file:
path: "{{ item.path }}"
state: directory
owner: "{{ item.owner|default(panko_system_user_name) }}"
group: "{{ item.group|default(panko_system_group_name) }}"
mode: "{{ item.mode|default('0755') }}"
with_items:
- { path: "/var/www/cgi-bin", owner: root, group: root }
- { path: "/var/www/cgi-bin/panko" }
- { path: "/var/run/panko-api" }

22
templates/panko-api-wsgi.py.j2
View File

@ -1,22 +0,0 @@
# Copyright 2016 Comcast Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
activate_this = os.path.expanduser("{{ panko_bin }}/activate_this.py")
execfile(activate_this, dict(__file__=activate_this))
from panko.api import app
application = app.build_wsgi_app()

28
templates/panko-httpd.conf.j2
View File

@ -1,28 +0,0 @@
# {{ ansible_managed }}
<VirtualHost *:{{ panko_service_port }}>
WSGIDaemonProcess panko-api lang='en_US.UTF-8' locale='en_US.UTF-8' user={{ panko_system_user_name }} group={{ panko_system_group_name }} processes={{ panko_wsgi_processes }} threads={{ panko_wsgi_threads }} display-name=%{GROUP}
WSGIProcessGroup panko-api
WSGIScriptAlias / /var/www/cgi-bin/panko/panko-api
WSGIApplicationGroup %{GLOBAL}
<IfVersion >= 2.4>
ErrorLogFormat "%{cu}t %M"
</IfVersion>
LogLevel {{ panko_apache_log_level }}
ErrorLog /var/log/panko/panko-apache-error.log
CustomLog /var/log/panko/panko-access.log combined
</VirtualHost>
<Directory "/var/www/cgi-bin">
AllowOverride None
Options +ExecCGI -Includes
<IfModule !mod_access_compat.c>
Require all granted
</IfModule>
<IfModule mod_access_compat.c>
Order allow,deny
Allow from all
</IfModule>
</Directory>

3
templates/panko-ports.conf.j2
View File

@ -1,3 +0,0 @@
# {{ ansible_managed }}
Listen {{ panko_service_port }}

25
templates/panko-uwsgi.ini.j2 Normal file
View File

@ -0,0 +1,25 @@
[uwsgi]
uid = {{ panko_system_user_name }}
gid = {{ panko_system_group_name }}
virtualenv = {{ panko_bin | dirname }}
wsgi-file = {{ panko_bin }}/panko-api
http-socket = {{ panko_service_address }}:{{ panko_service_port }}
master = true
enable-threads = true
processes = {{ panko_wsgi_processes }}
threads = {{ panko_wsgi_threads }}
exit-on-reload = true
die-on-term = true
lazy-apps = true
add-header = Connection: close
buffer-size = 65535
thunder-lock = true
disable-logging = true
http-auto-chunked = true
http-raw-body = true
socket-timeout = 10
# Avoid filling up the logs with health check requests from haproxy.
route-user-agent = ^osa-haproxy-healthcheck$ donotlog:

21
vars/debian.yml
View File

@ -19,9 +19,6 @@ cache_timeout: 600
# Common apt packages
panko_distro_packages:
- rpcbind
- apache2
- apache2-utils
- libapache2-mod-wsgi
- libffi-dev
- libssl-dev
- libxml2-dev
@ -31,18 +28,10 @@ panko_devel_distro_packages:
- git
- libsystemd-dev
# NOTE(noonedeadpunk): These variables are left for migration from apache to uwsgi
# which was introduced during train release. They can be dropped afterwards.
panko_system_service_name: apache2
panko_apache_config:
- { src: "panko-ports.conf.j2", dest: "/etc/apache2/ports.conf" }
- { src: "panko-httpd.conf.j2", dest: "/etc/apache2/sites-available/panko-httpd.conf" }
panko_apache_default_sites:
- "/etc/apache2/sites-enabled/000-default.conf"
panko_apache_site_available: "/etc/apache2/sites-available/panko-httpd.conf"
panko_apache_site_enabled: "/etc/apache2/sites-enabled/panko-httpd.conf"
panko_apache_conf: "/etc/apache2/apache2.conf"
panko_apache_security_conf: "/etc/apache2/conf-available/security.conf"
- "/etc/apache2/sites-available/panko-httpd.conf"
- "/etc/apache2/sites-enabled/panko-httpd.conf"
panko_apache_ports: "/etc/apache2/ports.conf"

19
vars/redhat-7.yml
View File

@ -16,9 +16,6 @@
# Common packages
panko_distro_packages:
- rpcbind
- httpd
- httpd-tools
- mod_wsgi
- libffi-devel
- openssl-devel
- libxml2-devel
@ -29,17 +26,9 @@ panko_devel_distro_packages:
- git
- systemd-devel
# NOTE(noonedeadpunk): These variables are left for migration from apache to uwsgi
# which was introduced during train release. They can be dropped afterwards.
panko_system_service_name: httpd
panko_apache_config:
- { src: "panko-ports.conf.j2", dest: "/etc/httpd/conf.d/ports.conf" }
- { src: "panko-httpd.conf.j2", dest: "/etc/httpd/conf.d/panko-httpd.conf" }
panko_apache_default_sites:
- "/etc/httpd/conf.d/userdir.conf"
- "/etc/httpd/conf.d/welcome.conf"
- "/etc/httpd/conf.d/ssl.conf"
panko_apache_conf: "/etc/httpd/conf/httpd.conf"
panko_apache_security_conf: "{{ panko_apache_conf }}"
- "/etc/httpd/conf.d/panko-httpd.conf"
panko_apache_ports: "/etc/httpd/conf.d/ports.conf"

24
vars/suse.yml
View File

@ -17,9 +17,6 @@
# Common packages
panko_distro_packages:
- rpcbind
- apache2
- apache2-utils
- apache2-mod_wsgi
- libffi-devel
- libopenssl-devel
- libxml2-devel
@ -30,22 +27,9 @@ panko_devel_distro_packages:
- git-core
- systemd-devel
# NOTE(noonedeadpunk): These variables are left for migration from apache to uwsgi
# which was introduced during train release. They can be dropped afterwards.
panko_system_service_name: apache2
panko_apache_config:
- { src: "panko-ports.conf.j2", dest: "/etc/apache2/conf.d/ports.conf" }
- { src: "panko-httpd.conf.j2", dest: "/etc/apache2/conf.d/panko-httpd.conf" }
panko_apache_default_sites:
- "/etc/apache2/conf.d/gitweb.conf"
panko_apache_conf: "/etc/apache2/httpd.conf"
panko_apache_security_conf: "{{ panko_apache_conf }}"
panko_apache_modules:
- name: "authz_host"
state: "present"
- name: "access_compat"
state: "present"
- name: "version"
state: "present"
- "/etc/apache2/conf.d/panko-httpd.conf"
panko_apache_ports: /etc/apache2/conf.d/ports.conf