From 524a0524177885fbaa47c4ed0e47f8f80e84b17d Mon Sep 17 00:00:00 2001 From: "Donovan Francesco (drifterza)" Date: Fri, 23 Sep 2016 20:03:43 +0200 Subject: [PATCH] Refactoring Monasca role to align more with OSA. - This patch is to provide monasca backend services and monasca api,log-api - Adding the use of dictionary for service group mappings. - Switching to use tested external roles for backend services. Change-Id: I7f6335f12a4b3a13acf6b644f16587ab1b46745f --- defaults/main.yml | 193 +++++++---- extras/conf.d/monasca.yml.example | 36 ++ extras/env.d/monasca.yml | 131 +++++++ extras/haproxy_monasca.yml | 16 + extras/os-monasca-install.yml | 63 ++++ extras/repo_packages/openstack_monasca.yml | 60 ++++ extras/user_secrets.yml | 6 + tasks/install_storm.yml => handlers/main.yml | 40 +-- meta/main.yml | 24 +- tasks/{install_kafka.yml => configure.yml} | 21 -- tasks/configure_graphana.yml | 51 --- tasks/configure_influxdb.yml | 49 --- tasks/configure_kafka.yml | 57 ---- tasks/configure_storm.yml | 47 --- tasks/configure_zookeeper.yml | 56 --- tasks/install.yml | 160 +++++++++ tasks/install_graphana.yml | 73 ---- tasks/main.yml | 57 ++-- tasks/monasca_init_common.yml | 29 ++ tasks/monasca_init_systemd.yml | 56 +++ .../monasca_init_upstart.yml | 34 +- tasks/monasca_service_setup.yml | 141 ++++++++ tasks/post-install.yml | 52 ++- tasks/pre-install.yml | 28 +- tasks/pre-monasca-install.yml | 69 ++++ templates/graphana/graphana-server.j2 | 144 -------- templates/graphana/graphana.ini.j2 | 13 - templates/influxdb/influxdb.conf.j2 | 322 ------------------ templates/kafka/kafka-server-start.sh.j2 | 38 --- templates/kafka/kafka.j2 | 34 -- templates/kafka/log4j.properties.j2 | 71 ---- templates/kafka/server.properties.j2 | 118 ------- templates/monasca-api-logging.conf.j2 | 46 +++ templates/monasca-api-paste.ini.j2 | 22 ++ templates/monasca-api.conf.j2 | 83 +++++ templates/monasca-log-api-logging.conf.j2 | 36 ++ templates/monasca-log-api-paste.ini.j2 | 21 ++ templates/monasca-log-api.conf.j2 | 38 +++ templates/monasca-systemd-init.j2 | 25 ++ templates/monasca-systemd-tempfiles.j2 | 4 + templates/monasca-upstart-init.j2 | 42 +++ templates/storm/cluster.xml.j2 | 85 ----- templates/storm/storm.yml.j2 | 58 ---- templates/zookeeper/environment.j2 | 35 -- templates/zookeeper/log4j.properties.j2 | 68 ---- templates/zookeeper/zoo.cfg.j2 | 73 ---- tests/ansible-role-requirements.yml | 45 ++- .../group_vars/all_containers.yml | 34 +- .../host_vars/localhost.yml | 15 +- tests/inventory | 38 +++ tests/test-install-monasca.yml | 106 ++++++ tests/test-monasca-functional.yml | 41 +++ tests/test-vars.yml | 63 ++++ tests/test.yml | 24 +- tox.ini | 13 - vars/ubuntu-14.04.yml | 10 - .../ubuntu-16.04.yml | 25 +- 57 files changed, 1698 insertions(+), 1641 deletions(-) create mode 100644 extras/conf.d/monasca.yml.example create mode 100644 extras/env.d/monasca.yml create mode 100644 extras/haproxy_monasca.yml create mode 100644 extras/os-monasca-install.yml create mode 100644 extras/repo_packages/openstack_monasca.yml create mode 100644 extras/user_secrets.yml rename tasks/install_storm.yml => handlers/main.yml (56%) rename tasks/{install_kafka.yml => configure.yml} (58%) delete mode 100644 tasks/configure_graphana.yml delete mode 100644 tasks/configure_influxdb.yml delete mode 100644 tasks/configure_kafka.yml delete mode 100644 tasks/configure_storm.yml delete mode 100644 tasks/configure_zookeeper.yml create mode 100644 tasks/install.yml delete mode 100644 tasks/install_graphana.yml create mode 100644 tasks/monasca_init_common.yml create mode 100644 tasks/monasca_init_systemd.yml rename templates/storm/storm-nimbus.conf.j2 => tasks/monasca_init_upstart.yml (52%) create mode 100644 tasks/monasca_service_setup.yml create mode 100644 tasks/pre-monasca-install.yml delete mode 100644 templates/graphana/graphana-server.j2 delete mode 100644 templates/graphana/graphana.ini.j2 delete mode 100644 templates/influxdb/influxdb.conf.j2 delete mode 100644 templates/kafka/kafka-server-start.sh.j2 delete mode 100644 templates/kafka/kafka.j2 delete mode 100644 templates/kafka/log4j.properties.j2 delete mode 100644 templates/kafka/server.properties.j2 create mode 100644 templates/monasca-api-logging.conf.j2 create mode 100644 templates/monasca-api-paste.ini.j2 create mode 100644 templates/monasca-api.conf.j2 create mode 100644 templates/monasca-log-api-logging.conf.j2 create mode 100644 templates/monasca-log-api-paste.ini.j2 create mode 100644 templates/monasca-log-api.conf.j2 create mode 100644 templates/monasca-systemd-init.j2 create mode 100644 templates/monasca-systemd-tempfiles.j2 create mode 100644 templates/monasca-upstart-init.j2 delete mode 100644 templates/storm/cluster.xml.j2 delete mode 100644 templates/storm/storm.yml.j2 delete mode 100644 templates/zookeeper/environment.j2 delete mode 100644 templates/zookeeper/log4j.properties.j2 delete mode 100644 templates/zookeeper/zoo.cfg.j2 rename templates/storm/storm-supervisor.conf.j2 => tests/group_vars/all_containers.yml (55%) rename templates/influxdb/influxdb.j2 => tests/host_vars/localhost.yml (72%) create mode 100644 tests/test-install-monasca.yml create mode 100644 tests/test-monasca-functional.yml create mode 100644 tests/test-vars.yml rename tasks/install_influxdb.yml => vars/ubuntu-16.04.yml (61%) diff --git a/defaults/main.yml b/defaults/main.yml index 93940b6..5546cde 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -15,78 +15,155 @@ # # (c) 2016 Donovan Francesco # (c) 2016 Paul Stevens -# -# -# Originally obtained from https://github.com/elastic/ansible-elasticsearch -# -kafka_directory: "/opt/kafka" +monasca_package_state: "latest" -## Monasca -monasca_common_git_repo: "https://git.openstack.org/openstack/monasca-common.git" -monasca_api_git_repo: "https://git.openstack.org/openstack/monasca-api.git" -monasca_persister_git_repo: "https://git.openstack.org/openstack/monasca-persister.git" -monasca_notification_git_repo: "https://git.openstack.org/openstack/monasca-notification.git" -monasca_thresh_git_repo: "https://git.openstack.org/openstack/monasca-thresh.git" -monasca_ui_git_repo: "https://git.openstack.org/openstack/monasca-ui.git" -monasca_client_git_repo: "https://git.openstack.org/openstack/python-monascaclient.git" -monasca_git_branch: "master" - -## Grafana -grafana_plugins_git_repo: "https://github.com/twc-openstack/grafana-plugins.git" -grafana_plugins_git_branch: "v2.6.0" -grafana_git_repo: "https://github.com/twc-openstack/grafana.git" -grafana_git_branch: "v2.6.0-keystone" - -## Keystone -keystone_ip_address: "127.0.0.1" -# admin credentials -os_username: "admin" -os_password: "secretadmin" -os_project_name: "admin" - -# monasca-user credentials -os_mon_username: "mini-mon" -os_mon_password: "password" -os_mon_project_name: "mini-mon" - -# monasca-agent credentials -os_mon_agent_username: "monasca-agent" -os_mon_agent_password: "password" - -## Database -mysql_root_pass: "secretdatabase" - -## InfluxDB -influxdb_version: "0.9.5" - -## Kafka -base_kafka_version: "0.10.0.0" -kafka_version: "kafka_2.11-{{ base_kafka_version }}" - -## Storm -storm_version: "1.0.1" -storm_logback_dir: "log4j2" - -## NTP -ntp_servers: - - ntp.is.co.za +debug: false ## System info monasca_system_user_name: monasca monasca_system_group_name: monasca -monasca_system_shell: /bin/false +monasca_system_user_shell: /bin/false monasca_system_comment: monasca system user monasca_system_user_home: "/var/lib/{{ monasca_system_user_name }}" +monasca_bin: "/openstack/venvs/monasca-{{ monasca_venv_tag }}/bin" +monasca_log_directory: "/var/log/monasca" +monasca_conf_directory: "/etc/monasca" + +monasca_service_region: RegionOne +monasca_service_host: "0.0.0.0" +monasca_service_port: 8070 +monasca_service_publicuri_proto: http +monasca_service_publicurl: "{{ monasca_service_publicuri_proto }}://{{ external_lb_vip_address }}:{{ monasca_service_port }}/v2.0" +monasca_service_internaluri_proto: http +monasca_service_internalurl: "{{ monasca_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ monasca_service_port }}/v2.0" +monasca_service_adminuri_proto: http +monasca_service_adminurl: "{{ monasca_service_adminuri_proto }}://{{ internal_lb_vip_address }}:{{ monasca_service_port }}/v2.0" +monasca_auth_url: "{{ keystone_service_internalurl }}" +monasca_keystone_auth_plugin: password +monasca_service_tenant_name: monasca +monasca_service_project_name: monasca +monasca_service_description: "Monasca Monitoring Service" +monasca_service_name: monasca +monasca_service_type: monitoring +monasca_agent_user_name: monasca-agent +monasca_regular_user_name: monasca-user +monasca_service_admin_role_names: admin +monasca_backend_database: "influxdb" + +monasca_use_mod_wsgi: false + +monasca_api_service_port: 8070 +monasca_api_git_repo: "https://git.openstack.org/openstack/monasca-api" +monasca_api_git_install_branch: master +monasca_api_requirements_git_repo: https://git.openstack.org/openstack/requirements +monasca_api_requirements_git_install_branch: master +monasca_api_metrics_driver: "monasca_api.common.repositories.influxdb.metrics_repository:MetricsRepository" + +monasca_events_api_service_port: 8072 +monasca_events_api_git_repo: "https://git.openstack.org/openstack/monasca-events-api" +monasca_events_api_git_install_branch: master +monasca_events_api_requirements_git_repo: https://git.openstack.org/openstack/requirements +monasca_events_api_requirements_git_install_branch: master + +monasca_log_api_service_port: 5607 +monasca_log_api_git_repo: "https://git.openstack.org/openstack/monasca-log-api" +monasca_log_api_git_install_branch: master +monasca_log_api_requirements_git_repo: https://git.openstack.org/openstack/requirements +monasca_log_api_requirements_git_install_branch: master + +monasca_ceilometer_git_repo: "https://git.openstack.org/openstack/monasca-ceilometer" +monasca_ceilometer_git_install_branch: master +monasca_ceilometer_requirements_git_repo: https://git.openstack.org/openstack/requirements +monasca_ceilometer_requirements_git_install_branch: master + +monasca_common_git_repo: "https://git.openstack.org/openstack/monasca-common" +monasca_common_git_install_branch: master +monasca_common_requirements_git_repo: https://git.openstack.org/openstack/requirements +monasca_common_requirements_git_install_branch: master + +monasca_thresh_git_repo: "https://git.openstack.org/openstack/monasca-thresh" +monasca_thresh_git_install_branch: master +monasca_thresh_requirements_git_repo: https://git.openstack.org/openstack/requirements +monasca_thresh_requirements_git_install_branch: master + +monasca_transform_git_repo: "https://git.openstack.org/openstack/monasca-transform" +monasca_transform_git_install_branch: master +monasca_transform_requirements_git_repo: https://git.openstack.org/openstack/requirements +monasca_transform_requirements_git_install_branch: master + +monasca_notification_git_repo: "https://git.openstack.org/openstack/monasca-notification" +monasca_notification_git_install_branch: master +monasca_notification_requirements_git_repo: https://git.openstack.org/openstack/requirements +monasca_notification_requirements_git_install_branch: master + +monasca_persister_git_repo: "https://git.openstack.org/openstack/monasca-persister" +monasca_persister_git_install_branch: master +monasca_persister_requirements_git_repo: https://git.openstack.org/openstack/requirements +monasca_persister_requirements_git_install_branch: master + +monasca_python_client_git_repo: "https://git.openstack.org/openstack/python-monascaclient" +monasca_python_client_git_install_branch: master +monasca_python_client_requirements_git_repo: https://git.openstack.org/openstack/requirements +monasca_python_client_requirements_git_install_branch: master + +monasca_developer_constraints: + - "git+{{ monasca_api_git_repo }}@{{ monasca_api_git_install_branch }}#egg=monasca-api" + - "git+{{ monasca_events_api_git_repo }}@{{ monasca_events_api_git_install_branch }}#egg=monasca-events-api" + - "git+{{ monasca_log_api_git_repo }}@{{ monasca_log_api_git_install_branch }}#egg=monasca-log-api" + - "git+{{ monasca_ceilometer_git_repo }}@{{ monasca_ceilometer_git_install_branch }}#egg=monasca-ceilometer" + - "git+{{ monasca_common_git_repo }}@{{ monasca_common_git_install_branch }}#egg=monasca-common" + - "git+{{ monasca_transform_git_repo }}@{{ monasca_transform_git_install_branch }}#egg=monasca-transform" + - "git+{{ monasca_notification_git_repo }}@{{ monasca_notification_git_install_branch }}#egg=monasca-notification" + - "git+{{ monasca_persister_git_repo }}@{{ monasca_persister_git_install_branch }}#egg=monasca-persister" + - "git+{{ monasca_python_client_git_repo }}@{{ monasca_python_client_git_install_branch }}#egg=python-monascaclient" + +# Name of the virtual env to deploy into +monasca_venv_tag: untagged +monasca_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/monasca.tgz + +monasca_grafana_galera_database: grafana +monasca_grafana_galera_username: grafana pip_install_options: "" -monasca_required_pip_packages: +monasca_services: + monasca-api: + group: monasca_api + service_name: monasca-api + service_init_bin: "{{ monasca_bin }}/gunicorn" + service_init_options: "-n monasca-api -k eventlet --worker-connections=2000 --backlog=1000 --paste /etc/monasca/api-config.ini" + monasca-log-api: + group: monasca_log_api + service_name: monasca-log-api + service_init_bin: "{{ monasca_bin }}/gunicorn" + service_init_options: "-n monasca-log-api -k eventlet --worker-connections=2000 --backlog=1000 --paste /etc/monasca/log-api-config.ini" + +monasca_requires_pip_packages: + - httplib2 + - python-glanceclient + - python-keystoneclient + - python-monascaclient + - python-memcached - virtualenv - virtualenv-tools - - httplib2 # Common pip packages monasca_pip_packages: - - python-keystoneclient - keystoneauth1 - simport + - gunicorn + - monasca-common + - monasca-api + - monasca-log-api + - monasca-persister + - monasca-transform + - monasca-notification + - influxdb + +monasca_api_paste_ini_overrides: {} +monasca_api_config_overrides: {} +monasca_log_api_paste_ini_overrides: {} +monasca_log_api_config_overrides: {} +monasca_log_api_logging_overrides: {} +monasca_notification_yml_overrides: {} +monasca_persister_yml_overrides: {} diff --git a/extras/conf.d/monasca.yml.example b/extras/conf.d/monasca.yml.example new file mode 100644 index 0000000..8638553 --- /dev/null +++ b/extras/conf.d/monasca.yml.example @@ -0,0 +1,36 @@ +# The infra nodes that will be running the monasca services +monasca-infra_hosts: + infra1: + ip: 172.20.236.111 + infra2: + ip: 172.20.236.112 + infra3: + ip: 172.20.236.113 +monasca-zookeeper_hosts: + infra1: + ip: 172.20.236.111 + infra2: + ip: 172.20.236.112 + infra3: + ip: 172.20.236.113 +monasca-kafka_hosts: + infra1: + ip: 172.20.236.111 + infra2: + ip: 172.20.236.112 + infra3: + ip: 172.20.236.113 +monasca-storm_hosts: + infra1: + ip: 172.20.236.111 + infra2: + ip: 172.20.236.112 + infra3: + ip: 172.20.236.113 +monasca-influxdb_hosts: + infra1: + ip: 172.20.236.111 + infra2: + ip: 172.20.236.112 + infra3: + ip: 172.20.236.113 diff --git a/extras/env.d/monasca.yml b/extras/env.d/monasca.yml new file mode 100644 index 0000000..f0d843b --- /dev/null +++ b/extras/env.d/monasca.yml @@ -0,0 +1,131 @@ +--- +# Copyright 2016 Internet Solutions (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2016 Donovan Francesco +# (c) 2016 Paul Stevens + +component_skel: + monasca_api: + belongs_to: + - monasca_all + monasca_events_api: + belongs_to: + - monasca_all + monasca_log_api: + belongs_to: + - monasca_all + + monasca_zookeeper: + belongs_to: + - monasca_all + monasca_kafka: + belongs_to: + - monasca_all + monasca_storm: + belongs_to: + - monasca_all + monasca_influxdb: + belongs_to: + - monasca_all + monasca_grafana: + belongs_to: + - monasca_all + +container_skel: + monasca_api_container: + belongs_to: + - monasca-infra_containers + contains: + - monasca_api + - monasca_common + properties: + service_name: monasca + monasca_events_api_container: + belongs_to: + - monasca-infra_containers + contains: + - monasca_events_api + - monasca_common + properties: + service_name: monasca + monasca_log_api_container: + belongs_to: + - monasca-infra_containers + contains: + - monasca_log_api + - monasca_common + properties: + service_name: monasca + + monasca_zookeeper_container: + belongs_to: + - monasca-zookeeper_containers + contains: + - monasca_zookeeper + properties: + service_name: monasca + monasca_kafka_container: + belongs_to: + - monasca-kafka_containers + contains: + - monasca_kafka + properties: + service_name: monasca + monasca_influxdb_container: + belongs_to: + - monasca-influxdb_containers + contains: + - monasca_influxdb + properties: + service_name: monasca + monasca_grafana_container: + belongs_to: + - monasca-infra_containers + contains: + - monasca_grafana + properties: + service_name: monasca + +physical_skel: + monasca-zookeeper_containers: + belongs_to: + - all_containers + monasca-kafka_containers: + belongs_to: + - all_containers + monasca-storm_containers: + belongs_to: + - all_containers + monasca-influxdb_containers: + belongs_to: + - all_containers + monasca-infra_containers: + belongs_to: + - all_containers + monasca-infra_hosts: + belongs_to: + - hosts + monasca-zookeeper_hosts: + belongs_to: + - hosts + monasca-kafka_hosts: + belongs_to: + - hosts + monasca-storm_hosts: + belongs_to: + - hosts + monasca-influxdb_hosts: + belongs_to: + - hosts diff --git a/extras/haproxy_monasca.yml b/extras/haproxy_monasca.yml new file mode 100644 index 0000000..4de17e7 --- /dev/null +++ b/extras/haproxy_monasca.yml @@ -0,0 +1,16 @@ + - service: + haproxy_service_name: monasca-api + haproxy_backend_nodes: "{{ groups['monasca_api'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: "{{ monasca_api_service_port }}" + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + - service: + haproxy_service_name: monasca-log-api + haproxy_backend_nodes: "{{ groups['monasca_log_api'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: "{{ monasca_log_api_service_port }}" + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" diff --git a/extras/os-monasca-install.yml b/extras/os-monasca-install.yml new file mode 100644 index 0000000..f070e9c --- /dev/null +++ b/extras/os-monasca-install.yml @@ -0,0 +1,63 @@ +--- +# Copyright 2016 Internet Solutions (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2016 Donovan Francesco +# (c) 2016 Paul Stevens + +- name: Install monasca server + hosts: monasca_all + max_fail_percentage: 20 + user: root + pre_tasks: + - include: common-tasks/os-lxc-container-setup.yml + - include: common-tasks/os-log-dir-setup.yml + vars: + log_dirs: + - src: "/openstack/log/{{ inventory_hostname }}-monasca" + dest: "/var/log/monasca" + - include: common-tasks/rabbitmq-vhost-user.yml + vars: + user: "{{ monasca_rabbitmq_userid }}" + password: "{{ monasca_rabbitmq_password }}" + vhost: "{{ monasca_rabbitmq_vhost }}" + when: + - inventory_hostname == groups['monasca_all'][0] + - groups['rabbitmq_all'] | length > 0 + - include: common-tasks/mysql-db-user.yml + vars: + user_name: "{{ monasca_galera_user }}" + password: "{{ monasca_galera_password }}" + login_host: "{{ monasca_galera_address }}" + db_name: "{{ monasca_galera_database }}" + when: inventory_hostname == groups['monasca_all'][0] + roles: + - role: "os_monasca" + monasca_venv_tag: "{{ openstack_release }}" + monasca_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/monasca-{{ openstack_release }}.tgz" + pip_lock_to_internal_repo: "{{ (pip_links | length) >= 1 }}" + tags: + - "os-monasca" + - { role: "openstack_openrc", tags: [ "openstack-openrc" ] } + - role: "rsyslog_client" + rsyslog_client_log_rotate_file: monasca_log_rotate + rsyslog_client_log_dir: "/var/log/monasca" + rsyslog_client_config_name: "99-monasca-rsyslog-client.conf" + tags: + - "monasca-rsyslog-client" + - role: "system_crontab_coordination" + tags: + - "system-crontab-coordination" + vars: + is_metal: "{{ properties.is_metal|default(false) }}" diff --git a/extras/repo_packages/openstack_monasca.yml b/extras/repo_packages/openstack_monasca.yml new file mode 100644 index 0000000..1e369e4 --- /dev/null +++ b/extras/repo_packages/openstack_monasca.yml @@ -0,0 +1,60 @@ +--- +# Copyright 2016 Internet Solutions (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2016 Donovan Francesco +# (c) 2016 Paul Stevens +monasca_api_git_repo: "https://git.openstack.org/openstack/monasca-api" +monasca_api_git_install_branch: 1e56736bba2e0e70f35e3ac69967872276e6ebcc # master as of 23/09/2016 +monasca_api_git_dest: "/opt/monasca_{{ monasca_api_git_install_branch | replace('/', '_') }}" + +monasca_events_api_git_repo: "https://git.openstack.org/openstack/monasca-events-api" +monasca_events_api_git_install_branch: dba30614943d6a003620336540a2774e238a567b # master as of 25/09/2016 +monasca_events_api_git_dest: "/opt/monasca_{{ monasca_events_api_git_install_branch | replace('/', '_') }}" + +monasca_log_api_git_repo: "https://git.openstack.org/openstack/monasca-log-api" +monasca_log_api_git_install_branch: 7175ff0c5595b2f119eb4b7cac1cf204dec175f4 # master as of 25/09/2016 +monasca_log_api_git_dest: "/opt/monasca_{{ monasca_log_api_git_install_branch | replace('/', '_') }}" + +monasca_ceilometer_git_repo: "https://git.openstack.org/openstack/monasca-ceilometer" +monasca_ceilometer_git_install_branch: c0f807294919cf6add64ec1f99f1b82e5a6330c3 # master as of 23/09/2016 +monasca_ceilometer_git_dest: "/opt/monasca_{{ monasca_ceilometer_git_install_branch | replace('/', '_') }}" + +monasca_common_git_repo: "https://git.openstack.org/openstack/monasca-common" +monasca_common_git_install_branch: da9fa8ca015efbac2248f017704c33cf83f6be5d # master as of 23/09/2016 +monasca_common_git_dest: "/opt/monasca_{{ monasca_api_git_install_branch | replace('/', '_') }}" + +monasca_thresh_git_repo: "https://git.openstack.org/openstack/monasca-thresh" +monasca_thresh_git_install_branch: a62c27a165e61d350fc96432878bec81c09a1587 # master as of 23/09/2016 +monasca_thresh_git_dest: "/opt/monasca_{{ monasca_thresh_git_install_branch | replace('/', '_') }}" + +monasca_transform_git_repo: "https://git.openstack.org/openstack/monasca-transform" +monasca_transform_git_install_branch: 87a89604675f93028b9c4e985aba9f6a9a1e53a6 # master as of 23/09/2016 +monasca_transform_git_dest: "/opt/monasca_{{ monasca_transform_git_install_branch | replace('/', '_') }}" + +monasca_notification_git_repo: "https://git.openstack.org/openstack/monasca-notification" +monasca_notification_git_install_branch: 4c39e3ad678cf117252f304fc8507020a7e98c2a # master as of 23/09/2016 +monasca_notification_git_dest: "/opt/monasca_{{ monasca_notification_git_install_branch | replace('/', '_') }}" + +monasca_analytics_git_repo: "https://git.openstack.org/openstack/monasca-analytics" +monasca_analytics_git_install_branch: ecfc9d6b955032d46eef02bcb1326869b820b626 # master as of 23/09/2016 +monasca_analytics_git_dest: "/opt/monasca_{{ monasca_analytics_git_install_branch | replace('/', '_') }}" + +monasca_persister_git_repo: "https://git.openstack.org/openstack/monasca-persister" +monasca_persister_git_install_branch: 96fbd64a146f89b60317418d0aedb9bfafc14f0e # master as of 23/09/2016 +monasca_persister_git_dest: "/opt/monasca_{{ monasca_persister_git_install_branch | replace('/', '_') }}" + +monasca_python_client_git_repo: "https://git.openstack.org/openstack/python-monascaclient" +monasca_python_client_git_install_branch: 4146e7e433d7c88ab5655625085ac21dfb994291 # master as of 25/09/2016 +monasca_python_client_git_dest: "/opt/monasca_{{ monasca_python_client_git_install_branch | replace('/', '_') }}" diff --git a/extras/user_secrets.yml b/extras/user_secrets.yml new file mode 100644 index 0000000..384e85b --- /dev/null +++ b/extras/user_secrets.yml @@ -0,0 +1,6 @@ +--- +monasca_galera_password: +monasca_rabbitmq_password: +monasca_service_password: +monasca_admin_user_password: +monasca_regular_user_password: diff --git a/tasks/install_storm.yml b/handlers/main.yml similarity index 56% rename from tasks/install_storm.yml rename to handlers/main.yml index 0db39ce..493252c 100644 --- a/tasks/install_storm.yml +++ b/handlers/main.yml @@ -15,24 +15,24 @@ # # (c) 2016 Donovan Francesco # (c) 2016 Paul Stevens -# -- name: Download Apache Storm - get_url: - url: "https://github.com/apache/storm/archive/{{ storm_version }}.tar.gz" - dest: "/root/apache-storm-{{ storm_version }}.tar.gz" - mode: 0600 - register: download_storm - until: download_storm | success - retries: 5 - delay: 3 -- name: Extract Apache Storm - unarchive: - src: "/root/apache-storm-{{ storm_version }}.tar.gz" - dest: "/opt" - creates: "/opt/apache-storm-{{ storm_version }}" - copy: "no" - register: extract_storm - until: extract_storm | success - retries: 5 - delay: 3 +- name: Reload systemd daemon + command: "systemctl daemon-reload" + notify: + - Restart monasca services + +- name: Reload upstart init scripts + shell: | + initctl reload-configuration + notify: + - Restart monasca services + +- name: Restart monasca services + service: + name: "{{ item.value.service_name }}" + state: "restarted" + pattern: "{{ item.value.service_name }}" + with_dict: "{{ monasca_services }}" + when: + - inventory_hostname in groups[item.value.group] + - "{{ item.value.condition | default(True) }}" diff --git a/meta/main.yml b/meta/main.yml index 814d3eb..0e14db9 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -20,15 +20,31 @@ allow_duplicates: yes galaxy_info: author: "Donovan Francesco, Paul Stevens" - description: "Apache Kafka for ECS" + description: "Openstack Monasca" company: "Internet Solutions" license: "license (Apache)" min_ansible_install: 1.9 platforms: - name: Ubuntu versions: - - all + - trusty + - xenial categories: - - system + - cloud + - python + - development + - openstack - dependencies: [] +dependencies: + - pip_install + - galera_client + - openstack_openrc + - role: apt_package_pinning + when: + - ansible_pkg_mgr == 'apt' + - ansible-zookeeper + - ansible-kafka + - ansible-storm + - ansible-influxdb + - ansible-grafana + - ansible-monasca-schema diff --git a/tasks/install_kafka.yml b/tasks/configure.yml similarity index 58% rename from tasks/install_kafka.yml rename to tasks/configure.yml index 1e82b69..1316168 100644 --- a/tasks/install_kafka.yml +++ b/tasks/configure.yml @@ -15,24 +15,3 @@ # # (c) 2016 Donovan Francesco # (c) 2016 Paul Stevens -# -- name: Download Apache Kafka - get_url: - url: http://apache.is.co.za/kafka/0.10.0.0/kafka_2.11-0.10.0.0.tgz - dest: /root/{{ kafka_version }} - mode: 0600 - register: download_kafka_binary - until: download_kafka_binary | success - retries: 5 - delay: 5 - -- name: Extract Apache Kafka archive - unarchive: - src: "/root/{{ kafka_version }}" - dest: "/opt" - creates: "/opt/kafka" - copy: "no" - register: extract_binary - until: extract_binary | success - retries: 5 - delay: 3 diff --git a/tasks/configure_graphana.yml b/tasks/configure_graphana.yml deleted file mode 100644 index 281fe2c..0000000 --- a/tasks/configure_graphana.yml +++ /dev/null @@ -1,51 +0,0 @@ ---- -# Copyright 2016 Internet Solutions (Pty) Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# (c) 2016 Donovan Francesco -# (c) 2016 Paul Stevens -# -- name: Copy Graphana Templates - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - owner: "{{ item.owner }}" - group: "{{ item.group }}" - mode: "0644" - register: configure_graphana - until: configure_graphana | success - retries: 5 - delay: 3 - with_items: - - src: "grafana/grafana-server.j2" - dest: "/etc/init.d/grafana-server" - owner: "root" - group: "root" - mode: "0775" - - src: "grafana/grafana.ini.j2" - dest: "/etc/grafana/grafana.ini" - owner: "root" - group: "root" - -- name: Setup Graphana to run at boot - command: update-rc.d graphana-server defaults 95 10 - -- name: Start the Graphana Service - service: - name: graphana-server - state: restarted - register: start_graphana - until: start_graphana | success - retries: 5 - delay: 3 diff --git a/tasks/configure_influxdb.yml b/tasks/configure_influxdb.yml deleted file mode 100644 index 59fb7bc..0000000 --- a/tasks/configure_influxdb.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- -# Copyright 2016 Internet Solutions (Pty) Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# (c) 2016 Donovan Francesco -# (c) 2016 Paul Stevens -# -- name: Copy InfluxDB Templates - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - owner: "{{ item.owner }}" - group: "{{ item.group }}" - mode: "0644" - with_items: - - src: "influxdb/influxdb.conf.j2" - dest: "/etc/influxdb/influxdb.conf" - owner: "root" - group: "root" - - src: "influxdb/influxdb.j2" - dest: "/etc/default/influxdb" - owner: "root" - group: "root" - register: configure_influxdb - until: configure_influxdb | success - retries: 5 - delay: 3 - -- name: Restart influxdb - service: - name: "influxdb" - state: "restarted" - when: influxdb_config.changed - -- name: Sleep for 60 seconds to let Influxdb elect a leader - pause: - minutes: 1 - when: influxdb_config.changed diff --git a/tasks/configure_kafka.yml b/tasks/configure_kafka.yml deleted file mode 100644 index 98ba4bf..0000000 --- a/tasks/configure_kafka.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -# Copyright 2016 Internet Solutions (Pty) Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# (c) 2016 Donovan Francesco -# (c) 2016 Paul Stevens -# -- name: Copy Kafka templates - template: - src: "{{ item.name }}" - dest: "{{ item.dest }}" - owner: "{{ item.owner }}" - group: "{{ item.group }}" - mode: "{{ item.mode | default('0644') }}" - with_items: - - src: "kafka/kafka-server-start.sh.j2" - dest: "/opt/kafka/bin/kafka-server-start.sh" - owner: "root" - group: "root" - mode: "0755" - - src: "kafka/kafka.j2" - dest: "/etc/init/kafka" - owner: "root" - group: "root" - - src: "kafka/log4j.properties.j2" - dest: "/opt/kafka/config/log4j.properties" - owner: "kafka" - group: "kafka" - - src: "kafka/server.properties.j2" - dest: "/opt/kafka/config/server.properties" - owner: "kafka" - group: "kafka" - register: copy_templates - until: copy_templates | success - retries: 5 - delay: 3 - -- name: Start Kafka - service: - name: "kafka" - state: "restarted" - when: kafka.config.changed - register: restart_kafka - until: restart_kafka | success - retries: 5 - delay: 3 diff --git a/tasks/configure_storm.yml b/tasks/configure_storm.yml deleted file mode 100644 index c057a87..0000000 --- a/tasks/configure_storm.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -# Copyright 2016 Internet Solutions (Pty) Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# (c) 2016 Donovan Francesco -# (c) 2016 Paul Stevens -# -- name: Copy Apache Storm Templates - template: - src: "{{ item.name }}" - dest: "{{ item.dest }}" - owner: "{{ item.owner }}" - group: "{{ item.group }}" - mode: "{{ item.mode | default('0644') }}" - with_items: - - src: "storm/cluster.xml.j2" - dest: "/opt/storm/{{ storm_logback_dir }}/cluster.xml" - owner: "storm" - group: "storm" - mode: "0644" - - src: "storm/storm.yaml.j2" - dest: "/opt/apache-storm-{{ storm_version }}/conf/storm.yaml" - owner: "storm" - group: "storm" - - src: "storm/storm-nimbus.conf.j2" - dest: "/etc/init/storm-nimbus.conf" - owner: "root" - group: "root" - - src: "storm/storm-supervisor.conf.j2" - dest: "/etc/init/storm-supervisor.conf" - owner: "root" - group: "root" - register: storm_config_templates - until: storm_config_templates | success - retries: 5 - delay: 3 diff --git a/tasks/configure_zookeeper.yml b/tasks/configure_zookeeper.yml deleted file mode 100644 index 4b194a4..0000000 --- a/tasks/configure_zookeeper.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -# Copyright 2016 Internet Solutions (Pty) Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# (c) 2016 Donovan Francesco -# (c) 2016 Paul Stevens -# -- name: Copy Zookeeper Templates - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - owner: "{{ item.owner }}" - group: "{{ item.group }}" - mode: "0644" - with_items: - - src: "zookeeper/zoo.cfg.j2" - dest: "/etc/zookeeper/conf/zoo.cfg" - owner: "root" - group: "root" - - src: "zookeeper/myid.j2" - dest: "/etc/zookeeper/conf/myid" - owner: "root" - group: "root" - - src: "zookeeper/environment.j2" - dest: "/etc/zookeeper/conf/environment" - owner: "root" - group: "root" - - src: "zookeeper/log4j.properties.j2" - dest: "/etc/zookeeper/conf/log4j.properties" - owner: "root" - group: "root" - register: configure_zookeeper - until: configure_zookeeper | success - retries: 5 - delay: 3 - -- name: Restart zookeeper - service: - name: "zookeeper" - state: "restarted" - when: zookeeper_config.changed - register: restart_zookeeper - until: restart_zookeeper | success - retries: 5 - delay: 3 diff --git a/tasks/install.yml b/tasks/install.yml new file mode 100644 index 0000000..fe38497 --- /dev/null +++ b/tasks/install.yml @@ -0,0 +1,160 @@ +--- +# Copyright 2016 Internet Solutions (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2016 Donovan Francesco +# (c) 2016 Paul Stevens + +- name: Create developer mode constraint file + copy: + dest: "/opt/developer-pip-constraints.txt" + content: | + {% for item in monasca_developer_constraints %} + {{ item }} + {% endfor %} + when: + - monasca_developer_mode | bool + +- name: Clone requirements git repository + git: + repo: "{{ item.repo }}" + dest: "/opt/requirements" + clone: yes + update: yes + version: "{{ item.version }}" + when: + - monasca_developer_mode | bool + with_items: + - repo: "{{ monasca_api_requirements_git_repo }}" + version: "{{ monasca_api_requirements_git_install_branch }}" + - repo: "{{ monasca_log_api_requirements_git_repo }}" + version: "{{ monasca_log_api_requirements_git_install_branch }}" + - repo: "{{ monasca_ceilometer_requirements_git_repo }}" + version: "{{ monasca_ceilometer_requirements_git_install_branch }}" + - repo: "{{ monasca_common_requirements_git_repo }}" + version: "{{ monasca_common_requirements_git_install_branch }}" + - repo: "{{ monasca_transform_requirements_git_repo }}" + version: "{{ monasca_transform_requirements_git_install_branch }}" + - repo: "{{ monasca_notification_requirements_git_repo }}" + version: "{{ monasca_notification_requirements_git_install_branch }}" + - repo: "{{ monasca_persister_requirements_git_repo }}" + version: "{{ monasca_persister_requirements_git_install_branch }}" + - repo: "{{ monasca_python_client_requirements_git_repo }}" + version: "{{ monasca_python_client_requirements_git_install_branch }}" + +- name: Add constraints to pip_install_options fact for developer mode + set_fact: + pip_install_options_fact: "{{ pip_install_options|default('') }} --constraint /opt/developer-pip-constraints.txt --constraint /opt/requirements/upper-constraints.txt" + when: + - monasca_developer_mode | bool + +- name: Set pip_install_options_fact when not in developer mode + set_fact: + pip_install_options_fact: "{{ pip_install_options|default('') }}" + when: + - not monasca_developer_mode | bool + +- name: Install requires pip packages + pip: + name: "{{ monasca_requires_pip_packages | join(' ') }}" + state: latest + extra_args: "{{ pip_install_options_fact }}" + register: install_packages + until: install_packages|success + retries: 5 + delay: 2 + +- name: Get local venv checksum + stat: + path: "/var/cache/{{ monasca_venv_download_url | basename }}" + get_md5: False + when: + - not monasca_developer_mode | bool + register: local_venv_stat + +- name: Get remote venv checksum + uri: + url: "{{ monasca_venv_download_url | replace('tgz', 'checksum') }}" + return_content: True + when: + - not monasca_developer_mode | bool + register: remote_venv_checksum + +# TODO: When project moves to ansible 2 we can pass this a sha256sum which will: +# a) allow us to remove force: yes +# b) allow the module to calculate the checksum of dest file which would +# result in file being downloaded only if provided and dest sha256sum +# checksums differ +- name: Attempt venv download + get_url: + url: "{{ monasca_venv_download_url }}" + dest: "/var/cache/{{ monasca_venv_download_url | basename }}" + force: yes + ignore_errors: true + register: get_venv + when: + - not monasca_developer_mode | bool + - (local_venv_stat.stat.exists == False or + {{ local_venv_stat.stat.checksum is defined and local_venv_stat.stat.checksum != remote_venv_checksum.content | trim }}) + +- name: Set monasca get_venv fact + set_fact: + monasca_get_venv: "{{ get_venv }}" + +- name: Remove existing venv + file: + path: "{{ monasca_bin | dirname }}" + state: absent + when: + - monasca_get_venv | changed + +- name: Create monasca venv dir + file: + path: "{{ monasca_bin | dirname }}" + state: directory + register: monasca_venv_dir + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ monasca_venv_download_url | basename }}" + dest: "{{ monasca_bin | dirname }}" + copy: "no" + when: + - not monasca_developer_mode | bool + - monasca_get_venv | changed or monasca_venv_dir | changed + notify: + - Restart monasca services + +- name: Install pip packages + pip: + name: "{{ monasca_pip_packages | join(' ') }}" + state: latest + virtualenv: "{{ monasca_bin | dirname }}" + virtualenv_site_packages: "no" + extra_args: "{{ pip_install_options_fact }}" + register: install_packages + until: install_packages|success + retries: 5 + delay: 2 + when: + - monasca_get_venv | failed or monasca_developer_mode | bool + notify: + - Restart monasca services + +- name: Update virtualenv path + command: > + virtualenv-tools --update-path=auto {{ monasca_bin | dirname }} + when: + - not monasca_developer_mode | bool + - monasca_get_venv | success diff --git a/tasks/install_graphana.yml b/tasks/install_graphana.yml deleted file mode 100644 index e9885e8..0000000 --- a/tasks/install_graphana.yml +++ /dev/null @@ -1,73 +0,0 @@ ---- -# Copyright 2016 Internet Solutions (Pty) Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# (c) 2016 Donovan Francesco -# (c) 2016 Paul Stevens -# -- name: Download Go - get_url: - url: "https://storage.googleapis.com/golang/{{ graphana_golang_version }}.tar.gz" - dest: "/root/{{ graphana_golang_version }}.tar.gz" - mode: 0440 - register: download_go - until: download_go | success - retries: 5 - delay: 3 - -- name: Install Go - unarchive: - src: "/root/{{ graphana_golang_version }}.tar.gz" - dest: "/usr/local" - creates: "/usr/local/go" - copy: "no" - register: install_go - until: install_go | success - retries: 5 - delay: 3 - -- name: Setup Graphana Wheezy Repository - apt_repository: - repo: "deb https://packagecloud.io/grafana/stable/debian/ wheezy main" - state: "present" - register: add_graphana_apt - until: add_graphana_apt | success - retries: 5 - delay: 3 - -- name: Install Graphana Apt Key - apt_key: - url: "deb https://packagecloud.io/grafana/stable/debian/ wheezy main" - state: present - register: add_graphana_apt_key - until: add_graphana_apt_key | success - retries: 5 - delay: 3 - -- name: Update Apt repos - apt: - update_cache: yes - register: update_cache - until: update_cache | success - retries: 5 - delay: 3 - -- name: Install Graphana - apt: - pkg: graphana - state: latest - register: install_graphana - until: install_graphana | success - retries: 5 - delay: 3 diff --git a/tasks/main.yml b/tasks/main.yml index cb2b72d..e741ddc 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -27,46 +27,43 @@ tags: - always +- name: Check init system + command: cat /proc/1/comm + register: _pid1_name + tags: + - always + +- name: Set the name of pid1 + set_fact: + pid1_name: "{{ _pid1_name.stdout }}" + tags: + - always + - include: pre-install.yml tags: - os_monasca -- include: configure_zookeeper.yml +- include: install.yml tags: - os_monasca -- include: install_kafka.yml - tags: - - os_monasca - -- include: configure_kafka.yml - tags: - - os_monasca - -- include: install_influxdb.yml - tags: - - os_monasca - -- include: configure_influxdb.yml - tags: - - os_monasca - -- include: install_graphana.yml - tags: - - os_monasca - -- include: configure_graphana.yml - tags: - - os_monasca - -- include: install_storm.yml - tags: - - os_monasca - -- include: configure_storm.yml +- include: configure.yml tags: - os_monasca - include: post-install.yml tags: - os_monasca + +- include: monasca_service_setup.yml + static: no + when: inventory_hostname == groups['monasca_api'][0] + tags: + - monasca-install + +- include: monasca_init_common.yml + tags: + - os_monasca + +- name: Flush handlers + meta: flush_handlers diff --git a/tasks/monasca_init_common.yml b/tasks/monasca_init_common.yml new file mode 100644 index 0000000..32d2026 --- /dev/null +++ b/tasks/monasca_init_common.yml @@ -0,0 +1,29 @@ +--- +# Copyright 2016 Internet Solutions (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: monasca_init_upstart.yml + when: pid1_name == "init" + +- include: monasca_init_systemd.yml + when: pid1_name == "systemd" + +- name: Load service + service: + name: "{{ item.value.service_name }}" + enabled: "yes" + with_dict: "{{ monasca_services }}" + when: inventory_hostname in groups[item.value.group] + notify: + - Restart monasca services diff --git a/tasks/monasca_init_systemd.yml b/tasks/monasca_init_systemd.yml new file mode 100644 index 0000000..d621d80 --- /dev/null +++ b/tasks/monasca_init_systemd.yml @@ -0,0 +1,56 @@ +--- +# Copyright 2016 Internet Solutions (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create TEMP run dir + file: + path: "/var/run/{{ item.value.service_name }}" + state: directory + owner: "{{ monasca_system_user_name }}" + group: "{{ monasca_system_group_name }}" + mode: "02755" + with_dict: "{{ monasca_services }}" + when: inventory_hostname in groups[item.value.group] + +- name: Create TEMP lock dir + file: + path: "/var/lock/{{ item.value.service_name }}" + state: directory + owner: "{{ monasca_system_user_name }}" + group: "{{ monasca_system_group_name }}" + mode: "02755" + with_dict: "{{ monasca_services }}" + when: inventory_hostname in groups[item.value.group] + +- name: Create tempfile.d entry + template: + src: "monasca-systemd-tempfiles.j2" + dest: "/etc/tmpfiles.d/monasca.conf" + mode: "0644" + owner: "root" + group: "root" + with_dict: "{{ monasca_services }}" + when: inventory_hostname in groups[item.value.group] + +- name: Place the systemd init script + template: + src: "monasca-systemd-init.j2" + dest: "/etc/systemd/system/{{ item.value.service_name }}.service" + mode: "0644" + owner: "root" + group: "root" + with_dict: "{{ monasca_services }}" + when: inventory_hostname in groups[item.value.group] + notify: + - Reload systemd daemon diff --git a/templates/storm/storm-nimbus.conf.j2 b/tasks/monasca_init_upstart.yml similarity index 52% rename from templates/storm/storm-nimbus.conf.j2 rename to tasks/monasca_init_upstart.yml index b4ec769..3e5a4f7 100644 --- a/templates/storm/storm-nimbus.conf.j2 +++ b/tasks/monasca_init_upstart.yml @@ -1,30 +1,26 @@ -# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP +--- +# Copyright 2016 Internet Solutions (Pty) Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -#    http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Startup script for Storm Nimbus -description "Storm Nimbus daemon" -start on runlevel [2345] - -console log -respawn - -kill timeout 240 -respawn limit 25 5 - -setgid storm -setuid storm -chdir /opt/storm -exec /opt/storm/bin/storm nimbus +- name: Place the init script + template: + src: "monasca-upstart-init.j2" + dest: "/etc/init/{{ item.value.service_name }}.conf" + mode: "0644" + owner: "root" + group: "root" + with_dict: "{{ monasca_services }}" + when: inventory_hostname in groups[item.value.group] + notify: + - Reload upstart init scripts diff --git a/tasks/monasca_service_setup.yml b/tasks/monasca_service_setup.yml new file mode 100644 index 0000000..b2b1240 --- /dev/null +++ b/tasks/monasca_service_setup.yml @@ -0,0 +1,141 @@ +--- +# Copyright 2016 Internet Solutions (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2016 Donovan Francesco +# (c) 2016 Paul Stevens +- name: Ensure the monasca tenant exists + keystone: + command: "ensure_tenant" + endpoint: "{{ keystone_service_adminurl }}" + login_user: "{{ keystone_admin_user_name }}" + login_password: "{{ keystone_auth_admin_password }}" + login_project_name: "{{ keystone_admin_tenant_name }}" + insecure: "{{ keystone_service_adminuri_insecure }}" + tenant_name: "{{ monasca_service_tenant_name }}" + project_name: "{{ monasca_service_project_name }}" + description: "{{ monasca_service_description }}" + register: add_monasca_tenant + until: add_monasca_tenant |success + retries: 5 + delay: 2 + tags: + - monasca-api-setup + - monasca-service-add + - monasca-setup + +- name: Ensure the service for monasca exists + keystone: + command: "ensure_service" + endpoint: "{{ keystone_service_adminurl }}" + login_user: "{{ keystone_admin_user_name }}" + login_password: "{{ keystone_auth_admin_password }}" + login_project_name: "{{ keystone_admin_tenant_name }}" + insecure: "{{ keystone_service_adminuri_insecure }}" + service_name: "{{ monasca_service_name }}" + service_type: "{{ monasca_service_type }}" + description: "{{ monasca_service_description }}" + register: add_monasca_service + until: add_monasca_service |success + retries: 5 + delay: 2 + tags: + - monasca-api-setup + - monasca-service-add + - monasca-setup + +- name: Ensure the monasca-agent user exists + keystone: + command: "ensure_user" + endpoint: "{{ keystone_service_adminurl }}" + login_user: "{{ keystone_admin_user_name }}" + login_password: "{{ keystone_auth_admin_password }}" + login_project_name: "{{ keystone_admin_tenant_name }}" + insecure: "{{ keystone_service_adminuri_insecure }}" + user_name: "{{ monasca_agent_user_name }}" + tenant_name: "{{ monasca_service_tenant_name }}" + password: "{{ monasca_agent_user_password }}" + project_name: "{{ monasca_service_project_name }}" + register: add_monasca_agent_user + until: add_monasca_agent_user |success + retries: 5 + delay: 2 + tags: + - monasca-api-setup + - monasca-service-add + - monasca-user-add + - monasca-setup + +- name: Ensure the monasca regular user exists + keystone: + command: "ensure_user" + endpoint: "{{ keystone_service_adminurl }}" + login_user: "{{ keystone_admin_user_name }}" + login_password: "{{ keystone_auth_admin_password }}" + login_project_name: "{{ keystone_admin_tenant_name }}" + insecure: "{{ keystone_service_adminuri_insecure }}" + user_name: "{{ monasca_regular_user_name }}" + tenant_name: "{{ monasca_service_tenant_name }}" + password: "{{ monasca_regular_user_password }}" + project_name: "{{ monasca_service_project_name }}" + register: add_monasca_regular_user + until: add_monasca_regular_user |success + retries: 5 + delay: 2 + tags: + - monasca-api-setup + - monasca-service-add + - monasca-user-add + - monasca-setup + +- name: Ensure the monasca-agent user has the admin role + keystone: + command: "ensure_user_role" + endpoint: "{{ keystone_service_adminurl }}" + login_user: "{{ keystone_admin_user_name }}" + login_password: "{{ keystone_auth_admin_password }}" + login_project_name: "{{ keystone_admin_tenant_name }}" + user_name: "{{ monasca_agent_user_name }}" + tenant_name: "{{ monasca_service_tenant_name }}" + role_name: "{{ item }}" + insecure: "{{ keystone_service_adminuri_insecure }}" + project_name: "{{ monasca_service_project_name }}" + register: ensure_monasca_agent_roles + until: ensure_monasca_agent_roles |success + retries: 5 + delay: 2 + with_items: "{{ monasca_service_admin_role_names }}" + +- name: Ensure the monasca endpoint is registered + keystone: + command: "ensure_endpoint" + endpoint: "{{ keystone_service_adminurl }}" + login_user: "{{ keystone_admin_user_name }}" + login_password: "{{ keystone_auth_admin_password }}" + login_project_name: "{{ keystone_admin_tenant_name }}" + insecure: "{{ keystone_service_adminuri_insecure }}" + region_name: "{{ monasca_service_region }}" + service_name: "{{ monasca_service_name }}" + service_type: "{{ monasca_service_type }}" + endpoint_list: + - url: "{{ monasca_service_publicurl }}" + interface: "public" + - url: "{{ monasca_service_internalurl }}" + interface: "internal" + - url: "{{ monasca_service_adminurl }}" + interface: "admin" + register: add_monasca_endpoints + until: add_monasca_endpoints |success + retries: 5 + delay: 2 diff --git a/tasks/post-install.yml b/tasks/post-install.yml index 4255e7e..d4a3469 100644 --- a/tasks/post-install.yml +++ b/tasks/post-install.yml @@ -15,4 +15,54 @@ # # (c) 2016 Donovan Francesco # (c) 2016 Paul Stevens -# +- name: Drop monasca api Config(s) + config_template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ monasca_system_user_name }}" + group: "{{ monasca_system_group_name }}" + mode: "0644" + config_overrides: "{{ item.config_overrides }}" + config_type: "{{ item.config_type }}" + with_items: + - src: "monasca-api-paste.ini.j2" + dest: "/etc/monasca/api-config.ini" + config_overrides: "{{ monasca_api_paste_ini_overrides }}" + config_type: "ini" + - src: "monasca-api.conf.j2" + dest: "/etc/monasca/api-config.conf" + config_overrides: "{{ monasca_api_config_overrides }}" + config_type: "ini" + - src: "monasca-api-logging.conf.j2" + dest: "/etc/monasca/api-logging.conf" + config_overrides: "{{ monasca_api_config_overrides }}" + config_type: "ini" + notify: + - Restart monasca services + when: inventory_hostname in groups['monasca_api'] + +- name: Drop monasca-log-api Config(s) + config_template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ monasca_system_user_name }}" + group: "{{ monasca_system_group_name }}" + mode: "0644" + config_overrides: "{{ item.config_overrides }}" + config_type: "{{ item.config_type }}" + with_items: + - src: "monasca-log-api-paste.ini.j2" + dest: "/etc/monasca/log-api-config.ini" + config_overrides: "{{ monasca_log_api_paste_ini_overrides }}" + config_type: "ini" + - src: "monasca-log-api.conf.j2" + dest: "/etc/monasca/log-api-config.conf" + config_overrides: "{{ monasca_log_api_config_overrides }}" + config_type: "ini" + - src: "monasca-log-api-logging.conf.j2" + dest: "/etc/monasca/log-api-logging.conf" + config_overrides: "{{ monasca_log_api_logging_overrides }}" + config_type: "ini" + notify: + - Restart monasca services + when: inventory_hostname in groups['monasca_log_api'] diff --git a/tasks/pre-install.yml b/tasks/pre-install.yml index 82e672b..781b9d0 100644 --- a/tasks/pre-install.yml +++ b/tasks/pre-install.yml @@ -15,7 +15,6 @@ # # (c) 2016 Donovan Francesco # (c) 2016 Paul Stevens -# - name: Update Apt sources apt: update_cache: yes @@ -34,26 +33,7 @@ delay: 2 with_items: "{{ monasca_distro_packages }}" -- name: Add a kafka user with temp sudo permissions # sudo removed in post-install.yml - user: - name: kafka - comment: "Apache Kafka Service" - groups: sudo - append: yes - generate_ssh_key: yes - ssh_key_bits: 2048 - ssh_key_file: .ssh/id_rsa.pub - register: create_kafka_user - until: create_kafka_user | success - retries: 5 - delay: 2 - -- name: Create Kafka directory - file: - path: /opt/kafka - state: directory - mode: 0755 - register: create_install_dir - until: create_install_dir | success - retries: 5 - delay: 2 +- include: pre-monasca-install.yml + when: + - inventory_hostname in groups['monasca_api'] + - inventory_hostname in groups['monasca_log_api'] diff --git a/tasks/pre-monasca-install.yml b/tasks/pre-monasca-install.yml new file mode 100644 index 0000000..9e94f58 --- /dev/null +++ b/tasks/pre-monasca-install.yml @@ -0,0 +1,69 @@ +--- +# Copyright 2016 Internet Solutions (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2016 Donovan Francesco +# (c) 2016 Paul Stevens + +- name: Create the monasca system group + group: + name: "{{ monasca_system_group_name }}" + state: "present" + system: "yes" + +- name: Create the monasca system user + user: + name: "{{ monasca_system_user_name }}" + group: "{{ monasca_system_group_name }}" + groups: sudo + shell: "{{ monasca_system_user_shell }}" + system: "yes" + createhome: "yes" + home: "{{ monasca_system_user_home }}" + register: create_monasca_user + until: create_monasca_user | success + retries: 5 + delay: 2 + +- name: Create monasca's directories + file: + path: "{{ item.path }}" + state: "directory" + owner: "{{ item.owner |default(monasca_system_user_name) }}" + group: "{{ item.group |default(monasca_system_group_name) }}" + mode: "{{ item.mode |default('0750') }}" + with_items: + - path: "{{ monasca_conf_directory }}" + - path: "{{ monasca_system_user_home }}" + +- name: Test for monasca log directory or link + shell: | + if [ -h "{{ monasca_log_directory }}" ]; then + chown -h {{ monasca_system_user_name }}:{{ monasca_system_group_name }} {{ monasca_log_directory }} + chown -R {{ monasca_system_user_name }}:{{ monasca_system_group_name }} "$(readlink {{ monasca_log_directory }})" + else + exit 1 + fi + register: log_dir + failed_when: false + changed_when: log_dir.rc != 0 + +- name: Create monasca log directory + file: + path: "{{ monasca_log_directory }}" + state: "directory" + owner: "{{ monasca_system_user_name }}" + group: "{{ monasca_system_group_name }}" + mode: "0750" + when: log_dir.rc != 0 diff --git a/templates/graphana/graphana-server.j2 b/templates/graphana/graphana-server.j2 deleted file mode 100644 index 80d091b..0000000 --- a/templates/graphana/graphana-server.j2 +++ /dev/null @@ -1,144 +0,0 @@ -#! /usr/bin/env bash - -# chkconfig: 2345 80 05 -# description: Grafana web server & backend -# processname: grafana -# config: /etc/grafana/grafana.ini -# pidfile: /var/run/grafana.pid - -### BEGIN INIT INFO -# Provides: grafana -# Required-Start: $all -# Required-Stop: $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Start grafana at boot time -### END INIT INFO - -# tested on -# 1. New lsb that define start-stop-daemon -# 3. Centos with initscripts package installed - -PATH=/bin:/usr/bin:/sbin:/opt/grafana-build/src/github.com/grafana/grafana/bin -NAME=grafana-server -DESC="Grafana Server" -DEFAULT=/etc/default/$NAME - -GRAFANA_USER=grafana -GRAFANA_GROUP=grafana -GRAFANA_HOME=/opt/grafana-build/src/github.com/grafana/grafana -CONF_DIR=/etc/grafana -WORK_DIR=$GRAFANA_HOME -DATA_DIR=/var/lib/grafana -LOG_DIR=/var/log/grafana -CONF_FILE=$CONF_DIR/grafana.ini -MAX_OPEN_FILES=10000 -PID_FILE=/var/run/$NAME.pid -DAEMON=/opt/grafana-build/src/github.com/grafana/grafana/bin/$NAME - -umask 0027 - -if [ `id -u` -ne 0 ]; then - echo "You need root privileges to run this script" - exit 4 -fi - -if [ ! -x $DAEMON ]; then - echo "Program not installed or not executable" - exit 5 -fi - -. /lib/lsb/init-functions - -if [ -r /etc/default/rcS ]; then - . /etc/default/rcS -fi - -# overwrite settings from default file -if [ -f "$DEFAULT" ]; then - . "$DEFAULT" -fi - -DAEMON_OPTS="--pidfile=${PID_FILE} --config=${CONF_FILE} cfg:default.paths.data=${DATA_DIR} cfg:default.paths.logs=${LOG_DIR}" - -case "$1" in - start) - - log_daemon_msg "Starting $DESC" - - pid=`pidofproc -p $PID_FILE grafana` - if [ -n "$pid" ] ; then - log_begin_msg "Already running." - log_end_msg 0 - exit 0 - fi - - # Prepare environment - mkdir -p "$LOG_DIR" "$DATA_DIR" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$LOG_DIR" "$DATA_DIR" - touch "$PID_FILE" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$PID_FILE" - - if [ -n "$MAX_OPEN_FILES" ]; then - ulimit -n $MAX_OPEN_FILES - fi - - # Start Daemon - start-stop-daemon --start -b --chdir "$WORK_DIR" --user "$GRAFANA_USER" -c "$GRAFANA_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS - return=$? - if [ $return -eq 0 ]; then - sleep 1 - - # check if pid file has been written two - if ! [[ -s $PID_FILE ]]; then - log_end_msg 1 - exit 1 - fi - - i=0 - timeout=10 - # Wait for the process to be properly started before exiting - until { cat "$PID_FILE" | xargs kill -0; } >/dev/null 2>&1; do - sleep 1 - i=$(($i + 1)) - if [ $i -gt $timeout ]; then - log_end_msg 1 - exit 1 - fi - done - fi - log_end_msg $return - ;; - stop) - log_daemon_msg "Stopping $DESC" - - if [ -f "$PID_FILE" ]; then - start-stop-daemon --stop --pidfile "$PID_FILE" \ - --user "$GRAFANA_USER" \ - --retry=TERM/20/KILL/5 >/dev/null - if [ $? -eq 1 ]; then - log_progress_msg "$DESC is not running but pid file exists, cleaning up" - elif [ $? -eq 3 ]; then - PID="`cat $PID_FILE`" - log_failure_msg "Failed to stop $DESC (pid $PID)" - exit 1 - fi - rm -f "$PID_FILE" - else - log_progress_msg "(not running)" - fi - log_end_msg 0 - ;; - status) - status_of_proc -p $PID_FILE grafana grafana && exit 0 || exit $? - ;; - restart|force-reload) - if [ -f "$PID_FILE" ]; then - $0 stop - sleep 1 - fi - $0 start - ;; - *) - log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}" - exit 3 - ;; -esac diff --git a/templates/graphana/graphana.ini.j2 b/templates/graphana/graphana.ini.j2 deleted file mode 100644 index 279039a..0000000 --- a/templates/graphana/graphana.ini.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[auth.keystone] -enabled = true -auth_url = http://{{ internal_lb_vip_address }}:5000 -[database] -type = mysql -host = {{ internal_lb_vip_address }}:3306 -name = graphana -user = graphana -password = {{ graphana_mysql_password }} -ssl_mode = skip-verify -[security] -admin_user = admin -admin_password = {{ graphana_admin_password }} diff --git a/templates/influxdb/influxdb.conf.j2 b/templates/influxdb/influxdb.conf.j2 deleted file mode 100644 index a697357..0000000 --- a/templates/influxdb/influxdb.conf.j2 +++ /dev/null @@ -1,322 +0,0 @@ -### Welcome to the InfluxDB configuration file. - -# Once every 24 hours InfluxDB will report anonymous data to m.influxdb.com -# The data includes raft id (random 8 bytes), os, arch, version, and metadata. -# We don't track ip addresses of servers reporting. This is only used -# to track the number of instances running and the versions, which -# is very helpful for us. -# Change this option to true to disable reporting. -reporting-disabled = false - -### -### Enterprise registration control -### - -[registration] -# enabled = true -# url = "https://enterprise.influxdata.com" # The Enterprise server URL -# token = "" # Registration token for Enterprise server - -### -### [meta] -### -### Controls the parameters for the Raft consensus group that stores metadata -### about the InfluxDB cluster. -### - -[meta] - dir = "/var/lib/influxdb/meta" - hostname = "{{ ansible_default_ipv4.address }}" - bind-address = ":8089" - retention-autocreate = true - election-timeout = "1s" - heartbeat-timeout = "1s" - leader-lease-timeout = "500ms" - commit-timeout = "50ms" - cluster-tracing = false - - # If enabled, when a Raft cluster loses a peer due to a `DROP SERVER` command, - # the leader will automatically ask a non-raft peer node to promote to a raft - # peer. This only happens if there is a non-raft peer node available to promote. - # This setting only affects the local node, so to ensure if operates correctly, be sure to set - # it in the config of every node. - raft-promotion-enabled = true - -### -### [data] -### -### Controls where the actual shard data for InfluxDB lives and how it is -### flushed from the WAL. "dir" may need to be changed to a suitable place -### for your system, but the WAL settings are an advanced configuration. The -### defaults should work for most systems. -### - -[data] - dir = "/var/lib/influxdb/data" - - # Controls the engine type for new shards. Options are b1, bz1, or tsm1. - # b1 is the 0.9.2 storage engine, bz1 is the 0.9.3 and 0.9.4 engine. - # tsm1 is the 0.9.5 engine and is currently EXPERIMENTAL. Until 0.9.5 is - # actually released data written into a tsm1 engine may be need to be wiped - # between upgrades. - # engine ="bz1" - - # The following WAL settings are for the b1 storage engine used in 0.9.2. They won't - # apply to any new shards created after upgrading to a version > 0.9.3. - max-wal-size = 104857600 # Maximum size the WAL can reach before a flush. Defaults to 100MB. - wal-flush-interval = "10m" # Maximum time data can sit in WAL before a flush. - wal-partition-flush-delay = "2s" # The delay time between each WAL partition being flushed. - - # These are the WAL settings for the storage engine >= 0.9.3 - wal-dir = "/var/lib/influxdb/wal" - wal-enable-logging = true - - # When a series in the WAL in-memory cache reaches this size in bytes it is marked as ready to - # flush to the index - # wal-ready-series-size = 25600 - - # Flush and compact a partition once this ratio of series are over the ready size - # wal-compaction-threshold = 0.6 - - # Force a flush and compaction if any series in a partition gets above this size in bytes - # wal-max-series-size = 2097152 - - # Force a flush of all series and full compaction if there have been no writes in this - # amount of time. This is useful for ensuring that shards that are cold for writes don't - # keep a bunch of data cached in memory and in the WAL. - # wal-flush-cold-interval = "10m" - - # Force a partition to flush its largest series if it reaches this approximate size in - # bytes. Remember there are 5 partitions so you'll need at least 5x this amount of memory. - # The more memory you have, the bigger this can be. - # wal-partition-size-threshold = 20971520 - - # Whether queries should be logged before execution. Very useful for troubleshooting, but will - # log any sensitive data contained within a query. - # query-log-enabled = true - -### -### [hinted-handoff] -### -### Controls the hinted handoff feature, which allows nodes to temporarily -### store queued data when one node of a cluster is down for a short period -### of time. -### - -[hinted-handoff] - enabled = true - dir = "/var/lib/influxdb/hh" - max-size = 1073741824 - max-age = "168h" - retry-rate-limit = 0 - - # Hinted handoff will start retrying writes to down nodes at a rate of once per second. - # If any error occurs, it will backoff in an exponential manner, until the interval - # reaches retry-max-interval. Once writes to all nodes are successfully completed the - # interval will reset to retry-interval. - retry-interval = "1s" - retry-max-interval = "1m" - - # Interval between running checks for data that should be purged. Data is purged from - # hinted-handoff queues for two reasons. 1) The data is older than the max age, or - # 2) the target node has been dropped from the cluster. Data is never dropped until - # it has reached max-age however, for a dropped node or not. - purge-interval = "1h" - -### -### [cluster] -### -### Controls non-Raft cluster behavior, which generally includes how data is -### shared across shards. -### - -[cluster] - shard-writer-timeout = "10s" # The time within which a shard must respond to write. - write-timeout = "5s" # The time within which a write operation must complete on the cluster. - -### -### [retention] -### -### Controls the enforcement of retention policies for evicting old data. -### - -[retention] - enabled = true - check-interval = "30m" - -### -### [shard-precreation] -### -### Controls the precreation of shards, so they are created before data arrives. -### Only shards that will exist in the future, at time of creation, are precreated. - -[shard-precreation] - enabled = true - check-interval = "10m" - advance-period = "30m" - -### -### Controls the system self-monitoring, statistics and diagnostics. -### -### The internal database for monitoring data is created automatically if -### if it does not already exist. The target retention within this database -### is called 'monitor' and is also created with a retention period of 7 days -### and a replication factor of 1, if it does not exist. In all cases the -### this retention policy is configured as the default for the database. - -[monitor] - store-enabled = true # Whether to record statistics internally. - store-database = "_internal" # The destination database for recorded statistics - store-interval = "10s" # The interval at which to record statistics - -### -### [admin] -### -### Controls the availability of the built-in, web-based admin interface. If HTTPS is -### enabled for the admin interface, HTTPS must also be enabled on the [http] service. -### - -[admin] - enabled = true - bind-address = ":8083" - https-enabled = false - https-certificate = "/etc/ssl/influxdb.pem" - -### -### [http] -### -### Controls how the HTTP endpoints are configured. These are the primary -### mechanism for getting data into and out of InfluxDB. -### - -[http] - enabled = true - bind-address = ":8086" - auth-enabled = false - log-enabled = true - write-tracing = false - pprof-enabled = false - https-enabled = false - https-certificate = "/etc/ssl/influxdb.pem" - -### -### [[graphite]] -### -### Controls one or many listeners for Graphite data. -### - -[[graphite]] - enabled = false - # database = "graphite" - # bind-address = ":2003" - # protocol = "tcp" - # consistency-level = "one" - # name-separator = "." - - # These next lines control how batching works. You should have this enabled - # otherwise you could get dropped metrics or poor performance. Batching - # will buffer points in memory if you have many coming in. - - # batch-size = 1000 # will flush if this many points get buffered - # batch-pending = 5 # number of batches that may be pending in memory - # batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit - # udp-read-buffer = 0 # UDP Read buffer size, 0 means OS default. UDP listener will fail if set above OS max. - - ## "name-schema" configures tag names for parsing the metric name from graphite protocol; - ## separated by `name-separator`. - ## The "measurement" tag is special and the corresponding field will become - ## the name of the metric. - ## e.g. "type.host.measurement.device" will parse "server.localhost.cpu.cpu0" as - ## { - ## measurement: "cpu", - ## tags: { - ## "type": "server", - ## "host": "localhost, - ## "device": "cpu0" - ## } - ## } - # name-schema = "type.host.measurement.device" - - ## If set to true, when the input metric name has more fields than `name-schema` specified, - ## the extra fields will be ignored. - ## Otherwise an error will be logged and the metric rejected. - # ignore-unnamed = true - -### -### [collectd] -### -### Controls the listener for collectd data. -### - -[collectd] - enabled = false - # bind-address = "" - # database = "" - # typesdb = "" - - # These next lines control how batching works. You should have this enabled - # otherwise you could get dropped metrics or poor performance. Batching - # will buffer points in memory if you have many coming in. - - # batch-size = 1000 # will flush if this many points get buffered - # batch-pending = 5 # number of batches that may be pending in memory - # batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit - # read-buffer = 0 # UDP Read buffer size, 0 means OS default. UDP listener will fail if set above OS max. - -### -### [opentsdb] -### -### Controls the listener for OpenTSDB data. -### - -[opentsdb] - enabled = false - # bind-address = ":4242" - # database = "opentsdb" - # retention-policy = "" - # consistency-level = "one" - # tls-enabled = false - # certificate= "" - - # These next lines control how batching works. You should have this enabled - # otherwise you could get dropped metrics or poor performance. Only points - # metrics received over the telnet protocol undergo batching. - - # batch-size = 1000 # will flush if this many points get buffered - # batch-pending = 5 # number of batches that may be pending in memory - # batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit - -### -### [[udp]] -### -### Controls the listeners for InfluxDB line protocol data via UDP. -### - -[[udp]] - enabled = false - # bind-address = "" - # database = "udp" - # retention-policy = "" - - # These next lines control how batching works. You should have this enabled - # otherwise you could get dropped metrics or poor performance. Batching - # will buffer points in memory if you have many coming in. - - # batch-size = 1000 # will flush if this many points get buffered - # batch-pending = 5 # number of batches that may be pending in memory - # batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit - # read-buffer = 0 # UDP Read buffer size, 0 means OS default. UDP listener will fail if set above OS max. - -### -### [continuous_queries] -### -### Controls how continuous queries are run within InfluxDB. -### - -[continuous_queries] - log-enabled = true - enabled = true - recompute-previous-n = 2 - recompute-no-older-than = "10m" - compute-runs-per-interval = 10 - compute-no-more-than = "2m" diff --git a/templates/kafka/kafka-server-start.sh.j2 b/templates/kafka/kafka-server-start.sh.j2 deleted file mode 100644 index 6eb9633..0000000 --- a/templates/kafka/kafka-server-start.sh.j2 +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ $# -lt 1 ]; then - echo "USAGE: $0 [-daemon] server.properties" - exit 1 -fi - -base_dir=$(dirname $0) -export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" -export KAFKA_HEAP_OPTS="-Xms256m -Xmx256m" - -EXTRA_ARGS="-name kafkaServer -loggc" - -COMMAND=$1 -case $COMMAND in - -daemon) - EXTRA_ARGS="-daemon "$EXTRA_ARGS - shift - ;; - *) - ;; -esac - -exec $base_dir/kafka-run-class.sh $EXTRA_ARGS kafka.Kafka $@ diff --git a/templates/kafka/kafka.j2 b/templates/kafka/kafka.j2 deleted file mode 100644 index 49d0456..0000000 --- a/templates/kafka/kafka.j2 +++ /dev/null @@ -1,34 +0,0 @@ -# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -description "Kafka" - -start on runlevel [2345] -stop on runlevel [!2345] - -respawn - -limit nofile 32768 32768 - -# If zookeeper is running on this box also give it time to start up properly -pre-start script - if [ -e /etc/init.d/zookeeper ]; then - /etc/init.d/zookeeper start || true - fi -end script - -# Rather than using setuid/setgid sudo is used because the pre-start task must run as root -exec sudo -Hu kafka -g kafka KAFKA_HEAP_OPTS="-Xmx128m" /opt/kafka/bin/kafka-server-start.sh /etc/kafka/server.properties diff --git a/templates/kafka/log4j.properties.j2 b/templates/kafka/log4j.properties.j2 deleted file mode 100644 index 0a03b66..0000000 --- a/templates/kafka/log4j.properties.j2 +++ /dev/null @@ -1,71 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=WARN, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.kafkaAppender=org.apache.log4j.RollingFileAppender -log4j.appender.kafkaAppender.MaxFileSize=50MB -log4j.appender.kafkaAppender.MaxBackupIndex=4 -log4j.appender.kafkaAppender.File=/var/log/kafka/server.log -log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.stateChangeAppender=org.apache.log4j.RollingFileAppender -log4j.appender.stateChangeAppender.MaxFileSize=50MB -log4j.appender.stateChangeAppender.MaxBackupIndex=4 -log4j.appender.stateChangeAppender.File=/var/log/kafka/state-change.log -log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.controllerAppender=org.apache.log4j.RollingFileAppender -log4j.appender.controllerAppender.MaxFileSize=50MB -log4j.appender.controllerAppender.MaxBackupIndex=4 -log4j.appender.controllerAppender.File=/var/log/kafka/controller.log -log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -# Turn on all our debugging info -#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender -#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender -#log4j.logger.kafka.perf=DEBUG, kafkaAppender -#log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender -#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG -log4j.logger.kafka=WARN, kafkaAppender - -# Tracing requests results in large logs -#log4j.appender.requestAppender=org.apache.log4j.RollingFileAppender -#log4j.appender.requestAppender.MaxFileSize=50MB -#log4j.appender.requestAppender.MaxBackupIndex=4 -#log4j.appender.requestAppender.File=/var/log/kafka/kafka-request.log -#log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout -#log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n -# -#log4j.logger.kafka.network.RequestChannel$=TRACE, requestAppender -#log4j.additivity.kafka.network.RequestChannel$=false -# -#log4j.logger.kafka.network.Processor=TRACE, requestAppender -#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender -#log4j.additivity.kafka.server.KafkaApis=false -#log4j.logger.kafka.request.logger=TRACE, requestAppender -#log4j.additivity.kafka.request.logger=false - -log4j.logger.kafka.controller=TRACE, controllerAppender -log4j.additivity.kafka.controller=false - -log4j.logger.state.change.logger=TRACE, stateChangeAppender -log4j.additivity.state.change.logger=false diff --git a/templates/kafka/server.properties.j2 b/templates/kafka/server.properties.j2 deleted file mode 100644 index 2d53e16..0000000 --- a/templates/kafka/server.properties.j2 +++ /dev/null @@ -1,118 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# see kafka.server.KafkaConfig for additional details and defaults - -############################# Server Basics ############################# - -# The id of the broker. This must be set to a unique integer for each broker. -broker.id=0 - -############################# Socket Server Settings ############################# - -# The address the socket server listens on. It will get the value returned from -# java.net.InetAddress.getCanonicalHostName() if not configured. -# FORMAT: -# listeners = security_protocol://host_name:port -# EXAMPLE: -# listeners = PLAINTEXT://your.host.name:9092 -#listeners=PLAINTEXT://:9092 - -# Hostname and port the broker will advertise to producers and consumers. If not set, -# it uses the value for "listeners" if configured. Otherwise, it will use the value -# returned from java.net.InetAddress.getCanonicalHostName(). -#advertised.listeners=PLAINTEXT://your.host.name:9092 - -# The number of threads handling network requests -num.network.threads=3 - -# The number of threads doing disk I/O -num.io.threads=8 - -# The send buffer (SO_SNDBUF) used by the socket server -socket.send.buffer.bytes=102400 - -# The receive buffer (SO_RCVBUF) used by the socket server -socket.receive.buffer.bytes=102400 - -# The maximum size of a request that the socket server will accept (protection against OOM) -socket.request.max.bytes=104857600 - -########################### Queue Basics ############################# - -delete.topic.enable=true - -############################# Log Basics ############################# - -# A comma seperated list of directories under which to store log files -log.dirs=/tmp/kafka-logs - -# The default number of log partitions per topic. More partitions allow greater -# parallelism for consumption, but this will also result in more files across -# the brokers. -num.partitions=1 - -# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. -# This value is recommended to be increased for installations with data dirs located in RAID array. -num.recovery.threads.per.data.dir=1 - -############################# Log Flush Policy ############################# - -# Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. -# There are a few important trade-offs here: -# 1. Durability: Unflushed data may be lost if you are not using replication. -# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. -# The settings below allow one to configure the flush policy to flush data after a period of time or -# every N messages (or both). This can be done globally and overridden on a per-topic basis. - -# The number of messages to accept before forcing a flush of data to disk -log.flush.interval.messages=10000 - -# The maximum amount of time a message can sit in a log before we force a flush -log.flush.interval.ms=1000 - -############################# Log Retention Policy ############################# - -# The following configurations control the disposal of log segments. The policy can -# be set to delete segments after a period of time, or after a given size has accumulated. -# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens -# from the end of the log. - -# The minimum age of a log file to be eligible for deletion -log.retention.hours=168 - -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. -#log.retention.bytes=1073741824 - -# The maximum size of a log segment file. When this size is reached a new log segment will be created. -log.segment.bytes=1073741824 - -# The interval at which log segments are checked to see if they can be deleted according -# to the retention policies -log.retention.check.interval.ms=300000 - -############################# Zookeeper ############################# - -# Zookeeper connection string (see zookeeper docs for details). -# This is a comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". -# You can also append an optional chroot string to the urls to specify the -# root directory for all kafka znodes. -zookeeper.connect=localhost:2181 - -# Timeout in ms for connecting to zookeeper -zookeeper.connection.timeout.ms=6000 diff --git a/templates/monasca-api-logging.conf.j2 b/templates/monasca-api-logging.conf.j2 new file mode 100644 index 0000000..1f97733 --- /dev/null +++ b/templates/monasca-api-logging.conf.j2 @@ -0,0 +1,46 @@ +[loggers] +keys = root, sqlalchemy, kafka + +[handlers] +keys = console, file + +[formatters] +keys = generic + +[logger_root] +level = DEBUG +formatter = default +handlers = console, file + +[logger_sqlalchemy] +qualname = sqlalchemy.engine +# "level = INFO" logs SQL queries. +# "level = DEBUG" logs SQL queries and results. +# "level = WARN" logs neither. (Recommended for production systems.) +level = DEBUG +formatter = default +handlers = console, file +propagate=0 + +[logger_kafka] +qualname = kafka +level = DEBUG +formatter = default +handlers = console, file +propagate = 0 + +[handler_console] +class = logging.StreamHandler +args = (sys.stderr,) +level = DEBUG +formatter = generic + +[handler_file] +class = logging.handlers.RotatingFileHandler +level = DEBUG +formatter = generic +# store up to 5*100MB of logs +args = ('/var/log/{{ monasca_service_name }}/monasca-api.log', 'a', 104857600, 5) + +[formatter_generic] +format = %(asctime)s %(levelname)s [%(name)s][%(threadName)s] %(message)s diff --git a/templates/monasca-api-paste.ini.j2 b/templates/monasca-api-paste.ini.j2 new file mode 100644 index 0000000..72ee380 --- /dev/null +++ b/templates/monasca-api-paste.ini.j2 @@ -0,0 +1,22 @@ +[DEFAULT] +name = monasca_api + +[pipeline:main] +# Add validator in the pipeline so the metrics messages can be validated. +pipeline = auth keystonecontext api + +[app:api] +paste.app_factory = monasca_api.api.server:launch + +[filter:auth] +paste.filter_factory = keystonemiddleware.auth_token:filter_factory + +[filter:keystonecontext] +paste.filter_factory = monasca_api.middleware.keystone_context_filter:filter_factory + +[server:main] +use = egg:gunicorn#main +host = 0.0.0.0 +port = {{ monasca_api_service_port }} +workers = 1 +proc_name = monasca_api diff --git a/templates/monasca-api.conf.j2 b/templates/monasca-api.conf.j2 new file mode 100644 index 0000000..3a85848 --- /dev/null +++ b/templates/monasca-api.conf.j2 @@ -0,0 +1,83 @@ +[DEFAULT] +debug={{ debug }} +log_config_append=/etc/monasca/api-logging.conf +region = {{ keystone_service_region }} + +[dispatcher] +versions = monasca_api.v2.reference.versions:Versions +version_2_0 = monasca_api.v2.reference.version_2_0:Version2 +metrics = monasca_api.v2.reference.metrics:Metrics +metrics_measurements = monasca_api.v2.reference.metrics:MetricsMeasurements +metrics_statistics = monasca_api.v2.reference.metrics:MetricsStatistics +metrics_names = monasca_api.v2.reference.metrics:MetricsNames +alarm_definitions = monasca_api.v2.reference.alarm_definitions:AlarmDefinitions +alarms = monasca_api.v2.reference.alarms:Alarms +alarms_count = monasca_api.v2.reference.alarms:AlarmsCount +alarms_state_history = monasca_api.v2.reference.alarms:AlarmsStateHistory +notification_methods = monasca_api.v2.reference.notifications:Notifications +dimension_values = monasca_api.v2.reference.metrics:DimensionValues +dimension_names = monasca_api.v2.reference.metrics:DimensionNames +notification_method_types = monasca_api.v2.reference.notificationstype:NotificationsType + +[security] +default_authorized_roles = user, domainuser, domainadmin, {{ monasca_regular_user_name }} +agent_authorized_roles = {{ monasca_agent_user_name }} +read_only_authorized_roles = monasca-read-only-user +delegate_authorized_roles = admin + +[messaging] +driver = monasca_api.common.messaging.kafka_publisher:KafkaPublisher + +[repositories] +metrics_driver = {{ monasca_api_metrics_driver }} +alarm_definitions_driver = monasca_api.common.repositories.sqla.alarm_definitions_repository:AlarmDefinitionsRepository +alarms_driver = monasca_api.common.repositories.sqla.alarms_repository:AlarmsRepository +notifications_driver = monasca_api.common.repositories.sqla.notifications_repository:NotificationsRepository +notification_method_type_driver = monasca_api.common.repositories.sqla.notification_method_type_repository:NotificationMethodTypeRepository + +[dispatcher] +driver = v2_reference + +[kafka] +uri = {{ kafka_hosts }} +metrics_topic = metrics +group = api +max_retry = 1 +wait_time = 1 +async = False +compact = False +partitions = 0 + +{% if monasca_backend_database == 'influxdb' %} +[influxdb] +ip_address = {{ influxdb_host }} +port = {{ influxdb_port }} +user = {{ influxdb_admin }} +password = {{ influxdb_admin_password }} +database_name = {{ influxdb_dbname }} +{% elif monasca_backend_database == 'cassandra' %} +[cassandra] +cluster_ip_addresses: 127.0.0.1 +keyspace: monasca +{% else %} +[mysql] +database_name = {{ monasca_galera_database }} +hostname = {{ monasca_galera_address }} +username = {{ monasca_galera_user }} +password = {{ monasca_galera_password }} +{% endif %} + +[database] +url = "mysql+pymysql://{{ monasca_galera_user }}:{{ monasca_galera_password }}@{{ monasca_galera_address }}/{{ monasca_galera_database }}" + +[keystone_authtoken] +insecure = {{ keystone_service_internaluri_insecure | bool }} +auth_plugin = {{ monasca_keystone_auth_plugin }} +auth_url = {{ keystone_service_adminuri }} +auth_uri = {{ keystone_service_internaluri }} +project_domain_id = {{ monasca_service_project_domain_id }} +user_domain_id = {{ monasca_service_user_domain_id }} +project_name = {{ monasca_service_project_name }} +username = {{ monasca_service_user_name }} +password = {{ monasca_service_password }} +region_name = {{ keystone_service_region }} diff --git a/templates/monasca-log-api-logging.conf.j2 b/templates/monasca-log-api-logging.conf.j2 new file mode 100644 index 0000000..9e1a3b7 --- /dev/null +++ b/templates/monasca-log-api-logging.conf.j2 @@ -0,0 +1,36 @@ +[loggers] +keys = root, kafka + +[handlers] +keys = console, file + +[formatters] +keys = generic + +[logger_root] +level = DEBUG +formatter = default +handlers = console, file + +[logger_kafka] +qualname = kafka +level = DEBUG +formatter = default +handlers = console, file +propagate = 0 + +[handler_console] +class = logging.StreamHandler +args = (sys.stderr,) +level = DEBUG +formatter = generic + +[handler_file] +class = logging.handlers.RotatingFileHandler +level = DEBUG +formatter = generic +# store up to 5*100MB of logs +args = ('/var/log/{{ monasca_service_name }}/monasca-log-api.log', 'a', 104857600, 5) + +[formatter_generic] +format = %(asctime)s %(levelname)s [%(name)s][%(threadName)s] %(message)s diff --git a/templates/monasca-log-api-paste.ini.j2 b/templates/monasca-log-api-paste.ini.j2 new file mode 100644 index 0000000..a3cc8af --- /dev/null +++ b/templates/monasca-log-api-paste.ini.j2 @@ -0,0 +1,21 @@ +[DEFAULT] +name = monasca_log_api + +[pipeline:main] +pipeline = auth roles api + +[app:api] +paste.app_factory = monasca_log_api.server:launch + +[filter:auth] +paste.filter_factory = monasca_log_api.healthcheck.keystone_protocol:filter_factory + +[filter:roles] +paste.filter_factory = monasca_log_api.middleware.role_middleware:RoleMiddleware.factory + +[server:main] +use = egg:gunicorn#main +host = 0.0.0.0 +port = {{ monasca_log_api_service_port }} +workers = 1 +proc_name = monasca_log_api diff --git a/templates/monasca-log-api.conf.j2 b/templates/monasca-log-api.conf.j2 new file mode 100644 index 0000000..ce26488 --- /dev/null +++ b/templates/monasca-log-api.conf.j2 @@ -0,0 +1,38 @@ +[DEFAULT] +log_config_append=/etc/monasca/log-api-logging.conf +debug={{ debug }} +[service] +max_log_size = 1048576 +region = {{ keystone_service_region }} + +[log_publisher] +topics = logs +kafka_url = {{ kafka_hosts }} + +[keystone_authtoken] +insecure = {{ keystone_service_internaluri_insecure | bool }} +auth_plugin = {{ monasca_keystone_auth_plugin }} +auth_url = {{ keystone_service_adminuri }} +auth_uri = {{ keystone_service_internaluri }} +project_domain_id = {{ monasca_service_project_domain_id }} +user_domain_id = {{ monasca_service_user_domain_id }} +project_name = {{ monasca_service_project_name }} +username = {{ monasca_service_user_name }} +password = {{ monasca_service_password }} +region_name = {{ keystone_service_region }} + +[kafka_healthcheck] +kafka_url = {{ kafka_hosts }} +kafka_topics = log + +[roles_middleware] +path = /v2.0/log +path = /v3.0/logs +default_roles = user, domainuser, domainadmin, {{ monasca_regular_user_name }} +agent_roles = {{ monasca_agent_user_name }}, admin + +[dispatcher] +logs = monasca_log_api.reference.v2.logs:Logs +logs_v3 = monasca_log_api.reference.v3.logs:Logs +versions = monasca_log_api.reference.versions:Versions +healthchecks = monasca_log_api.reference.healthchecks:HealthChecks diff --git a/templates/monasca-systemd-init.j2 b/templates/monasca-systemd-init.j2 new file mode 100644 index 0000000..ea0f455 --- /dev/null +++ b/templates/monasca-systemd-init.j2 @@ -0,0 +1,25 @@ +# {{ ansible_managed }} + +[Unit] +Description=monasca openstack service +After=syslog.target +After=network.target + +[Service] +Type=simple +User={{ monasca_system_user_name }} +Group={{ monasca_system_group_name }} + +{% if item.value.service_init_bin is defined %} +ExecStart={{ item.value.service_init_bin }} {{ item.value.service_init_options |default('') }} --log-file=/var/log/monasca/{{ item.value.service_name }}.log +{% else %} +ExecStart={{ monasca_bin }}/{{ item.value.service_name }} {{ program_config_options|default('') }} --log-file=/var/log/monasca/{{ item.value.service_name }}.log +{% endif %} + +# Give a reasonable amount of time for the server to start up/shut down +TimeoutSec=300 +Restart=on-failure +RestartSec=150 + +[Install] +WantedBy=multi-user.target diff --git a/templates/monasca-systemd-tempfiles.j2 b/templates/monasca-systemd-tempfiles.j2 new file mode 100644 index 0000000..055d540 --- /dev/null +++ b/templates/monasca-systemd-tempfiles.j2 @@ -0,0 +1,4 @@ +# {{ ansible_managed }} + +D /var/lock/{{ item.value.service_name }} 2755 {{ monasca_system_user_name }} {{ monasca_system_group_name }} +D /var/run/{{ item.value.service_name }} 2755 {{ monasca_system_user_name }} {{ monasca_system_group_name }} diff --git a/templates/monasca-upstart-init.j2 b/templates/monasca-upstart-init.j2 new file mode 100644 index 0000000..5072734 --- /dev/null +++ b/templates/monasca-upstart-init.j2 @@ -0,0 +1,42 @@ +# {{ ansible_managed }} + + +description "{{ item.value.service_name }}" +author "Kevin Carter " + +start on runlevel [2345] +stop on runlevel [016] + +respawn +respawn limit 10 5 + +# Set the RUNBIN environment variable +env RUNBIN="{{ monasca_bin }}/{{ item.value.service_name }}" +# Change directory to service users home +chdir "{{ monasca_system_user_home }}" + +# Pre start actions +pre-start script + mkdir -p "/var/run/{{ item.value.service_name }}" + chown {{ monasca_system_user_name }}:{{ monasca_system_group_name }} "/var/run/{{ item.value.service_name }}" + + mkdir -p "/var/lock/{{ item.value.service_name }}" + chown {{ monasca_system_user_name }}:{{ monasca_system_group_name }} "/var/lock/{{ item.value.service_name }}" + + . {{ monasca_bin }}/activate + +end script + +# Post stop actions +post-stop script + rm "/var/run/{{ item.value.service_name }}/{{ item.value.service_name }}.pid" +end script + +# Run the start up job +exec start-stop-daemon --start \ + --chuid {{ monasca_system_user_name }} \ + --make-pidfile \ + --pidfile /var/run/{{ item.value.service_name }}/{{ item.value.service_name }}.pid \ + --exec "{{ item.value.service_init_bin|default('$RUNBIN') }}" \ + -- {{ item.value.service_init_options|default('') }} \ + --log-file=/var/log/{{ monasca_service_name }}/{{ item.value.service_name }}.log diff --git a/templates/storm/cluster.xml.j2 b/templates/storm/cluster.xml.j2 deleted file mode 100644 index e5144d4..0000000 --- a/templates/storm/cluster.xml.j2 +++ /dev/null @@ -1,85 +0,0 @@ - - - - - - ${storm.home}/logs/${logfile.name} - - ${storm.home}/logs/${logfile.name}.%i - 1 - 9 - - - - 100MB - - - - %d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n - - - - - ${storm.home}/logs/access.log - - ${storm.home}/logs/access.log.%i - 1 - 9 - - - - 100MB - - - - %d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n - - - - - ${storm.home}/logs/metrics.log - - ${storm.home}/logs/metrics.log.%i - 1 - 9 - - - - 2MB - - - - %d %-8r %m%n - - - - - - - - - - - - - - - - - - diff --git a/templates/storm/storm.yml.j2 b/templates/storm/storm.yml.j2 deleted file mode 100644 index 686ea1e..0000000 --- a/templates/storm/storm.yml.j2 +++ /dev/null @@ -1,58 +0,0 @@ -# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -### base -java.library.path: "/usr/local/lib:/opt/local/lib:/usr/lib" -storm.local.dir: "/var/storm" - -### zookeeper.* -storm.zookeeper.servers: - - "{{ ansible_default_ipv4.address }}" -storm.zookeeper.port: 2181 -storm.zookeeper.retry.interval: 5000 -storm.zookeeper.retry.times: 60 -storm.zookeeper.root: /storm -storm.zookeeper.session.timeout: 3000 - -### supervisor.* configs are for node supervisors -supervisor.slots.ports: - - 6701 - - 6702 -supervisor.childopts: -Xmx256m - -### worker.* configs are for task workers -worker.childopts: -Xmx1280m -XX:+UseConcMarkSweepGC -Dcom.sun.management.jmxremote - -### nimbus.* configs are for the master -nimbus.host: "{{ ansible_default_ipv4.address }}" -nimbus.thrift.port: 6627 -nimbus.childopts: -Xmx256m - -### ui.* configs are for the master -ui.host: {{ ansible_default_ipv4.address }} -ui.port: 8088 -ui.childopts: -Xmx768m - -### drpc.* configs - -### transactional.* configs -transactional.zookeeper.servers: - - "{{ ansible_default_ipv4.address }}" -transactional.zookeeper.port: 2181 -transactional.zookeeper.root: /storm-transactional - -### topology.* configs are for specific executing storms -topology.acker.executors: 1 -topology.debug: False diff --git a/templates/zookeeper/environment.j2 b/templates/zookeeper/environment.j2 deleted file mode 100644 index bf18b0b..0000000 --- a/templates/zookeeper/environment.j2 +++ /dev/null @@ -1,35 +0,0 @@ -# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Modified from http://packages.ubuntu.com/saucy/zookeeperd -NAME=zookeeper -ZOOCFGDIR=/etc/zookeeper/conf - -# seems, that log4j requires the log4j.properties file to be in the classpath -CLASSPATH="$ZOOCFGDIR:/usr/share/java/jline.jar:/usr/share/java/log4j-1.2.jar:/usr/share/java/xercesImpl.jar:/usr/share/java/xmlParserAPIs.jar:/usr/share/java/netty.jar:/usr/share/java/slf4j-api.jar:/usr/share/java/slf4j-log4j12.jar:/usr/share/java/zookeeper.jar" - -ZOOCFG="$ZOOCFGDIR/zoo.cfg" -ZOO_LOG_DIR=/var/log/zookeeper -USER=$NAME -GROUP=$NAME -PIDDIR=/var/run/$NAME -PIDFILE=$PIDDIR/$NAME.pid -SCRIPTNAME=/etc/init.d/$NAME -JAVA=/usr/bin/java -ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain" -ZOO_LOG4J_PROP="INFO,ROLLINGFILE" -JMXLOCALONLY=false -JAVA_OPTS="" diff --git a/templates/zookeeper/log4j.properties.j2 b/templates/zookeeper/log4j.properties.j2 deleted file mode 100644 index 5c39819..0000000 --- a/templates/zookeeper/log4j.properties.j2 +++ /dev/null @@ -1,68 +0,0 @@ -# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# From http://packages.ubuntu.com/saucy/zookeeperd - -# ZooKeeper Logging Configuration -# - -# Format is " (, )+ - -log4j.rootLogger=${zookeeper.root.logger} - -# Example: console appender only -# log4j.rootLogger=INFO, CONSOLE - -# Example with rolling log file -#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE - -# Example with rolling log file and tracing -#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE - -# -# Log INFO level and above messages to the console -# -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -log4j.appender.CONSOLE.Threshold=INFO -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n - -# -# Add ROLLINGFILE to rootLogger to get log file output -# Log DEBUG level and above messages to a log file -log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender -log4j.appender.ROLLINGFILE.Threshold=WARN -log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/zookeeper.log - -# Max log file size of 10MB -log4j.appender.ROLLINGFILE.MaxFileSize=10MB -# uncomment the next line to limit number of backup files -#log4j.appender.ROLLINGFILE.MaxBackupIndex=10 - -log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout -log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n - - -# -# Add TRACEFILE to rootLogger to get log file output -# Log DEBUG level and above messages to a log file -log4j.appender.TRACEFILE=org.apache.log4j.FileAppender -log4j.appender.TRACEFILE.Threshold=TRACE -log4j.appender.TRACEFILE.File=${zookeeper.log.dir}/zookeeper_trace.log - -log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout -### Notice we are including log4j's NDC here (%x) -log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n diff --git a/templates/zookeeper/zoo.cfg.j2 b/templates/zookeeper/zoo.cfg.j2 deleted file mode 100644 index c6f89a6..0000000 --- a/templates/zookeeper/zoo.cfg.j2 +++ /dev/null @@ -1,73 +0,0 @@ -# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html - -# The number of milliseconds of each tick -tickTime=2000 -# The number of ticks that the initial -# synchronization phase can take -initLimit=10 -# The number of ticks that can pass between -# sending a request and getting an acknowledgement -syncLimit=5 -# the directory where the snapshot is stored. -dataDir=/var/lib/zookeeper -# Place the dataLogDir to a separate physical disc for better performance -# dataLogDir=/disk2/zookeeper - -# the port at which the clients will connect -clientPort=2181 - -# Maximum number of clients that can connect from one client -maxClientCnxns=60 - -# specify all zookeeper servers -# The fist port is used by followers to connect to the leader -# The second one is used for leader election - -server.0={{ ansible_default_ipv4.address }}:2888:3888 - -# To avoid seeks ZooKeeper allocates space in the transaction log file in -# blocks of preAllocSize kilobytes. The default block size is 64M. One reason -# for changing the size of the blocks is to reduce the block size if snapshots -# are taken more often. (Also, see snapCount). -#preAllocSize=65536 - -# Clients can submit requests faster than ZooKeeper can process them, -# especially if there are a lot of clients. To prevent ZooKeeper from running -# out of memory due to queued requests, ZooKeeper will throttle clients so that -# there is no more than globalOutstandingLimit outstanding requests in the -# system. The default limit is 1,000.ZooKeeper logs transactions to a -# transaction log. After snapCount transactions are written to a log file a -# snapshot is started and a new transaction log file is started. The default -# snapCount is 10,000. -#snapCount=1000 - -# If this option is defined, requests will be will logged to a trace file named -# traceFile.year.month.day. -#traceFile= - -# Leader accepts client connections. Default value is "yes". The leader machine -# coordinates updates. For higher update throughput at thes slight expense of -# read throughput the leader can be configured to not accept clients and focus -# on coordination. -#leaderServes=yes - -# Autopurge every hour to avoid using lots of disk in bursts -# Order of the next 2 properties matters. -# autopurge.snapRetainCount must be before autopurge.purgeInterval. -autopurge.snapRetainCount=3 -autopurge.purgeInterval=1 diff --git a/tests/ansible-role-requirements.yml b/tests/ansible-role-requirements.yml index a6cfe61..6062630 100644 --- a/tests/ansible-role-requirements.yml +++ b/tests/ansible-role-requirements.yml @@ -1,4 +1,3 @@ ---- - name: apt_package_pinning src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning scm: git @@ -11,6 +10,10 @@ src: https://git.openstack.org/openstack/openstack-ansible-memcached_server scm: git version: master +- name: openstack_hosts + src: https://git.openstack.org/openstack/openstack-ansible-openstack_hosts + scm: git + version: master - name: lxc_hosts src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts scm: git @@ -19,6 +22,10 @@ src: https://git.openstack.org/openstack/openstack-ansible-lxc_container_create scm: git version: master +- name: memcached_server + src: https://git.openstack.org/openstack/openstack-ansible-memcached_server + scm: git + version: master - name: galera_client src: https://git.openstack.org/openstack/openstack-ansible-galera_client scm: git @@ -31,3 +38,39 @@ src: https://git.openstack.org/openstack/openstack-ansible-rabbitmq_server scm: git version: master +- name: openstack_openrc + src: https://git.openstack.org/openstack/openstack-ansible-openstack_openrc + scm: git + version: master +- name: os_keystone + src: https://git.openstack.org/openstack/openstack-ansible-os_keystone + scm: git + version: master +- name: openstack_hosts + src: https://github.com/openstack/openstack-ansible-openstack_hosts + scm: git + version: master +- name: ansible-zookeeper + src: https://github.com/Chillisystems/ansible-zookeeper + scm: git + version: master +- name: ansible-kafka + src: https://github.com/Chillisystems/ansible-kafka + scm: git + version: master +- name: ansible-storm + src: https://github.com/Chillisystems/ansible-storm + scm: git + version: master +- name: ansible-influxdb + src: https://github.com/Chillisystems/ansible-influxdb + scm: git + version: master +- name: ansible-grafana + src: https://github.com/Chillisystems/ansible-grafana + scm: git + version: master +- name: ansible-monasca-schema + src: https://github.com/Chillisystems/ansible-monasca-schema + scm: git + version: master diff --git a/templates/storm/storm-supervisor.conf.j2 b/tests/group_vars/all_containers.yml similarity index 55% rename from templates/storm/storm-supervisor.conf.j2 rename to tests/group_vars/all_containers.yml index e548f48..f37df47 100644 --- a/templates/storm/storm-supervisor.conf.j2 +++ b/tests/group_vars/all_containers.yml @@ -1,30 +1,26 @@ -# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP +--- +# Copyright 2016, Rackspace US, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -#    http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Startup script for Storm Supervisor -description "Storm Supervisor daemon" -start on runlevel [2345] - -console log -respawn - -kill timeout 240 -respawn limit 25 5 - -setgid storm -setuid storm -chdir /opt/storm -exec /opt/storm/bin/storm supervisor +container_name: "{{ inventory_hostname }}" +container_networks: + management_address: + address: "{{ ansible_host }}" + bridge: "br-mgmt" + interface: "eth1" + netmask: "255.255.255.0" + type: "veth" +physical_host: localhost +properties: + service_name: "{{ inventory_hostname }}" diff --git a/templates/influxdb/influxdb.j2 b/tests/host_vars/localhost.yml similarity index 72% rename from templates/influxdb/influxdb.j2 rename to tests/host_vars/localhost.yml index ec24ffe..65ddeaa 100644 --- a/templates/influxdb/influxdb.j2 +++ b/tests/host_vars/localhost.yml @@ -1,18 +1,19 @@ -# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP +--- +# Copyright 2016, Rackspace US, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -#    http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Logging -STDERR="/var/log/influxdb/influxd.log" +bridges: + - "br-mgmt" + +ansible_python_interpreter: "/usr/bin/python2" diff --git a/tests/inventory b/tests/inventory index e69de29..ea96dd5 100644 --- a/tests/inventory +++ b/tests/inventory @@ -0,0 +1,38 @@ +[all] +localhost ansible_become=True +infra1 ansible_host=10.100.100.2 ansible_become=True ansible_user=root +openstack1 ansible_host=10.100.100.3 ansible_become=True ansible_user=root + +[all_containers] +infra1 +openstack1 + +[rabbitmq_all] +infra1 + +[galera_all] +infra1 + +[memcached_all] +infra1 + +[service_all:children] +rabbitmq_all +galera_all +memcached_all + +[keystone_all] +openstack1 + +[monasca_api] +openstack1 + +[monasca_events_api] +openstack1 + +[monasca_log_api] +openstack1 + +[monasca_all:children] +monasca_api +monasca_log_api diff --git a/tests/test-install-monasca.yml b/tests/test-install-monasca.yml new file mode 100644 index 0000000..543d33e --- /dev/null +++ b/tests/test-install-monasca.yml @@ -0,0 +1,106 @@ +--- +# Copyright 2016 Internet Solutions (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2016 Donovan Francesco +# (c) 2016 Paul Stevens + +- name: Install monasca server + hosts: monasca_all + max_fail_percentage: 20 + user: root + gather_facts: true + pre_tasks: + - name: Ensure rabbitmq vhost + rabbitmq_vhost: + name: "{{ monasca_rabbitmq_vhost }}" + state: "present" + delegate_to: "{{ hostvars[groups['rabbitmq_all'][0]]['ansible_host'] }}" + when: inventory_hostname == groups['monasca_all'][0] + - name: Ensure rabbitmq user + rabbitmq_user: + user: "{{ monasca_rabbitmq_userid }}" + password: "{{ monasca_rabbitmq_password }}" + vhost: "{{ monasca_rabbitmq_vhost }}" + configure_priv: ".*" + read_priv: ".*" + write_priv: ".*" + state: "present" + delegate_to: "{{ hostvars[groups['rabbitmq_all'][0]]['ansible_host'] }}" + when: inventory_hostname == groups['monasca_all'][0] + - name: Create DB for service + mysql_db: + login_user: "{{ galera_root_user }}" + login_password: "{{ galera_root_password }}" + login_host: "{{ monasca_galera_address }}" + name: "{{ item }}" + state: "present" + with_items: + - "{{ monasca_galera_database }}" + - "{{ monasca_grafana_galera_database }}" + delegate_to: "{{ hostvars[groups['galera_all'][0]]['ansible_host'] }}" + when: inventory_hostname == groups['monasca_all'][0] + - name: Grant access to the DB for the monasca service + mysql_user: + login_user: "{{ galera_root_user }}" + login_password: "{{ galera_root_password }}" + login_host: "{{ monasca_galera_address }}" + name: "{{ monasca_galera_database }}" + password: "{{ monasca_galera_password }}" + host: "{{ item }}" + state: "present" + priv: "{{ monasca_galera_database }}.*:ALL" + with_items: + - "localhost" + - "%" + delegate_to: "{{ hostvars[groups['galera_all'][0]]['ansible_host'] }}" + when: inventory_hostname == groups['monasca_all'][0] + - name: Grant access to the DB for the monasca grafana service + mysql_user: + login_user: "{{ galera_root_user }}" + login_password: "{{ galera_root_password }}" + login_host: "{{ monasca_galera_address }}" + name: "{{ monasca_grafana_galera_username }}" + password: "{{ monasca_grafana_galera_password }}" + host: "{{ item }}" + state: "present" + priv: "{{ monasca_grafana_galera_database }}.*:ALL" + with_items: + - "localhost" + - "%" + delegate_to: "{{ hostvars[groups['galera_all'][0]]['ansible_host'] }}" + when: inventory_hostname == groups['monasca_all'][0] + roles: + - role: ansible-zookeeper + tags: zookeeper + - role: ansible-kafka + tags: kafka + - role: ansible-storm + tags: storm + - role: ansible-influxdb + tags: influxdb + - role: ansible-grafana + grafana_mysql_host: "{{ monasca_galera_address }}" + grafana_admin_password: "{{ monasca_grafana_admin_password }}" + grafana_mysql_db: "{{ monasca_grafana_galera_database }}" + grafana_mysql_user: "{{ monasca_grafana_galera_username }}" + grafana_mysql_password: "{{ monasca_grafana_galera_password }}" + grafana_keystone_url: "{{ keystone_service_internaluri }}" + tags: grafana-server + - role: ansible-monasca-schema + tags: monasca-schema + - role: "os_monasca" + vars_files: + - common/test-vars.yml + - test-vars.yml diff --git a/tests/test-monasca-functional.yml b/tests/test-monasca-functional.yml new file mode 100644 index 0000000..6f5521b --- /dev/null +++ b/tests/test-monasca-functional.yml @@ -0,0 +1,41 @@ +--- +# Copyright 2016 Internet Solutions (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2016 Donovan Francesco +# (c) 2016 Paul Stevens +# Reference: https://github.com/openstack/monasca-api/blob/master/docs/monasca-api-spec.md + +- name: Playbook for functional testing monasca + hosts: monasca_api + user: root + gather_facts: false + tasks: + - name: Install httplib2 so we can use the uri module + pip: + name: httplib2 + - name: Check the monasca-api + uri: + url: "http://localhost:8070" + status_code: 401 + register: result + until: + - result is defined + - result.status is defined + - result.status == 401 + retries: 5 + delay: 10 + vars_files: + - common/test-vars.yml + - test-vars.yml diff --git a/tests/test-vars.yml b/tests/test-vars.yml new file mode 100644 index 0000000..4719a26 --- /dev/null +++ b/tests/test-vars.yml @@ -0,0 +1,63 @@ +--- +# Copyright 2016 Internet Solutions (Pty) Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2016 Donovan Francesco +# (c) 2016 Paul Stevens +monasca_developer_mode: True +monasca_galera_address: "{{ hostvars[groups['galera_all'][0]]['ansible_host'] }}" +monasca_galera_database: monasca +monasca_galera_user: monasca +monasca_galera_password: "secrete" +monasca_keystone_auth_plugin: password +monasca_rabbitmq_port: "{{ rabbitmq_port }}" +monasca_rabbitmq_servers: "{{ rabbitmq_servers }}" +monasca_rabbitmq_use_ssl: "{{ rabbitmq_use_ssl }}" +monasca_rabbitmq_password: "secrete" +monasca_rabbitmq_userid: monasca +monasca_rabbitmq_vhost: /monasca +monasca_api_requirements_git_install_branch: master +monasca_events_api_requirements_git_install_branch: master +monasca_log_api_requirements_git_install_branch: master +monasca_ceilometer_requirements_git_install_branch: master +monasca_common_requirements_git_install_branch: master +monasca_transform_requirements_git_install_branch: master +monasca_notification_requirements_git_install_branch: master +monasca_analytics_requirements_git_install_branch: master +monasca_persister_requirements_git_install_branch: master +monasca_python_client_requirements_git_install_branch: master +monasca_service_password: "secrete" +monasca_regular_user_password: "secrete" +monasca_agent_user_password: "secrete" +monasca_service_project_domain_id: default +monasca_service_project_name: service +monasca_service_region: RegionOne +monasca_service_user_domain_id: default +monasca_service_user_name: monasca +monasca_service_port: 8070 +monasca_venv_tag: untagged +monasca_bin: "/openstack/venvs/monasca-{{ monasca_venv_tag }}/bin" +storm_nimbus_enabled: true +storm_supervisor_enabled: true +nimbus_host: "{{ zookeeper_hosts }}" +monasca_mysql_host: "{{ monasca_galera_address }}" +monasca_mysql_admin: "{{ monasca_galera_user }}" +monasca_mysql_admin_password: "{{ monasca_galera_password }}" +monasca_mysql_db: "{{ monasca_galera_database }}" +influxdb_admin: "monasca" +influxdb_admin_password: "secrete" +monasca_grafana_admin_password: "secrete" +monasca_grafana_galera_password: "secrete" +monasca_grafana_galera_database: grafana +monasca_grafana_galera_username: grafana diff --git a/tests/test.yml b/tests/test.yml index fee930d..1a94aef 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -1,16 +1,32 @@ --- -# Copyright 2016, Comcast Corp. +# Copyright 2016 Internet Solutions (Pty) Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -- name: Gate stub - hosts: localhost +# +# (c) 2016 Donovan Francesco +# (c) 2016 Paul Stevens + +# Setup the host +- include: common/test-setup-host.yml + +# Install RabbitMQ/MariaDB +- include: common/test-install-infra.yml + +# Install keystone +- include: common/test-install-keystone.yml + +# Install monasca +- include: test-install-monasca.yml + +# Test monasca +- include: test-monasca-functional.yml diff --git a/tox.ini b/tox.ini index 6d3e4fa..ff5f5d1 100644 --- a/tox.ini +++ b/tox.ini @@ -102,20 +102,7 @@ commands = bash -c "{toxinidir}/tests/common/test-ansible-lint.sh" -[testenv:func_base] -# NOTE(odyssey4me): this target does not use constraints because -# it doesn't work in OpenStack-CI yet. Once that's fixed, we can -# drop the install_command. -install_command = - pip install -U --force-reinstall {opts} {packages} - - [testenv:functional] -# NOTE(odyssey4me): this target does not use constraints because -# it doesn't work in OpenStack-CI yet. Once that's fixed, we can -# drop the install_command. -install_command = - {[testenv:func_base]install_command} deps = {[testenv:ansible]deps} commands = diff --git a/vars/ubuntu-14.04.yml b/vars/ubuntu-14.04.yml index 6179365..c301348 100644 --- a/vars/ubuntu-14.04.yml +++ b/vars/ubuntu-14.04.yml @@ -16,20 +16,10 @@ # (c) 2016 Donovan Francesco # (c) 2016 Paul Stevens # -# Originally obtained from https://github.com/elastic/ansible-elasticsearch -# monasca_distro_packages: - build-essential - - chrony - - git - - libmysqlclient-dev - - mailutils - - mariadb-client - - maven - - nodejs - openjdk-7-jdk - python-software-properties - python-dev - python-mysqldb - python-pip - - zookeeperd diff --git a/tasks/install_influxdb.yml b/vars/ubuntu-16.04.yml similarity index 61% rename from tasks/install_influxdb.yml rename to vars/ubuntu-16.04.yml index 8c94c23..a504a53 100644 --- a/tasks/install_influxdb.yml +++ b/vars/ubuntu-16.04.yml @@ -16,21 +16,10 @@ # (c) 2016 Donovan Francesco # (c) 2016 Paul Stevens # - -- name: Download InfluxDB - get_url: - url: https://dl.influxdata.com/influxdb/releases/{{ influxdb_version }}.deb - dest: /root/{{ influxdb_version }}.deb - mode: 0600 - register: download_influxdb_deb - until: download_influxdb_deb | success - retries: 5 - delay: 5 - -- name: Install InfluxDB - apt: - deb: /root/{{ influxdb_version }}.deb - register: install_influxdb - until: install_influxdb | success - retries: 3 - delay: 2 +monasca_distro_packages: + - build-essential + - openjdk-8-jdk + - python-software-properties + - python-dev + - python-mysqldb + - python-pip