Merge "Support deploying the Monasca Agent"

ansible/group_vars/all.yml

@@ -259,6 +259,8 @@ mistral_api_port: "8989"
 
 monasca_api_port: "8070"
 monasca_log_api_port: "5607"
+monasca_agent_forwarder_port: "17123"
+monasca_agent_statsd_port: "8125"
 
 mongodb_port: "27017"
 mongodb_web_port: "28017"
@@ -564,6 +566,19 @@ enable_prometheus: "no"
 # Clean images options are specified here
 enable_destroy_images: "no"
 
+####################
+# Monasca options
+####################
+
+# The OpenStack username used by the Monasca Agent and the Fluentd Monasca
+# plugin to post logs and metrics from the control plane to Monasca.
+monasca_agent_user: "monasca-agent"
+
+# The OpenStack project to which the control plane logs and metrics are
+# tagged with. Only users with the monasca read only user role, or higher
+# can access these from the Monasca APIs.
+monasca_control_plane_project: "monasca_control_plane"
+
 ####################
 # Logging options
 ####################

ansible/inventory/all-in-one

@@ -85,6 +85,13 @@ control
 [qdrouterd:children]
 control
 
+[monasca-agent:children]
+compute
+control
+monitoring
+network
+storage
+
 [monasca:children]
 monitoring
 
@@ -431,6 +438,15 @@ murano
 murano
 
 # Monasca
+[monasca-agent-collector:children]
+monasca-agent
+
+[monasca-agent-forwarder:children]
+monasca-agent
+
+[monasca-agent-statsd:children]
+monasca-agent
+
 [monasca-api:children]
 monasca
 

ansible/inventory/multinode

@@ -110,6 +110,13 @@ control
 [qdrouterd:children]
 control
 
+[monasca-agent:children]
+compute
+control
+monitoring
+network
+storage
+
 [monasca:children]
 monitoring
 
@@ -440,6 +447,15 @@ murano
 murano
 
 # Monasca
+[monasca-agent-collector:children]
+monasca-agent
+
+[monasca-agent-forwarder:children]
+monasca-agent
+
+[monasca-agent-statsd:children]
+monasca-agent
+
 [monasca-api:children]
 monasca
 

ansible/roles/monasca/defaults/main.yml

@@ -103,6 +103,39 @@ monasca_services:
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla"
     dimensions: "{{ monasca_persister_dimensions }}"
+  monasca-agent-collector:
+    container_name: monasca_agent_collector
+    group: monasca-agent-collector
+    enabled: true
+    image: "{{ monasca_agent_image_full }}"
+    pid_mode: "host"
+    volumes:
+      - "{{ node_config_directory }}/monasca-agent-collector/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
+      - "/sys:/sys:ro"
+      - "/dev/disk/:/dev/disk:ro"
+    dimensions: "{{ monasca_agent_dimensions }}"
+  monasca-agent-statsd:
+    container_name: monasca_agent_statsd
+    group: monasca-agent-statsd
+    enabled: true
+    image: "{{ monasca_agent_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/monasca-agent-statsd/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
+    dimensions: "{{ monasca_agent_dimensions }}"
+  monasca-agent-forwarder:
+    container_name: monasca_agent_forwarder
+    group: monasca-agent-forwarder
+    enabled: true
+    image: "{{ monasca_agent_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/monasca-agent-forwarder/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
+    dimensions: "{{ monasca_agent_dimensions }}"
 
 ####################
 # Databases
@@ -142,6 +175,12 @@ monasca_periodic_notifications_topic: "60-seconds-notifications"
 # implemented it is not recommended to change this period.
 monasca_periodic_notifications_period: 60
 
+# Agent settings
+monasca_agent_max_buffer_size: 1000
+monasca_agent_backlog_send_rate: 1000
+monasca_agent_max_batch_size: 1000
+monasca_agent_check_frequency: 30
+
 # Processing pipeline threads. In a large scale deployment you will likely
 # want to tune these with finer precision. For example, if you have a very
 # high log throughput, the log metrics service consumer may require a
@@ -157,6 +196,10 @@ monasca_metric_pipeline_threads: 2
 monasca_install_type: "{{ kolla_install_type }}"
 monasca_tag: "{{ openstack_release }}"
 
+monasca_agent_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-agent"
+monasca_agent_tag: "{{ monasca_tag }}"
+monasca_agent_image_full: "{{ monasca_agent_image }}:{{ monasca_agent_tag }}"
+
 monasca_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-api"
 monasca_api_tag: "{{ monasca_tag }}"
 monasca_api_image_full: "{{ monasca_api_image }}:{{ monasca_api_tag }}"
@@ -181,6 +224,7 @@ monasca_persister_image: "{{ docker_registry ~ '/' if docker_registry else '' }}
 monasca_persister_tag: "{{ monasca_tag }}"
 monasca_persister_image_full: "{{ monasca_persister_image }}:{{ monasca_persister_tag }}"
 
+monasca_agent_dimensions: "{{ default_container_dimensions }}"
 monasca_api_dimensions: "{{ default_container_dimensions }}"
 monasca_log_api_dimensions: "{{ default_container_dimensions }}"
 monasca_log_transformer_dimensions: "{{ default_container_dimensions }}"
@@ -190,7 +234,6 @@ monasca_thresh_dimensions: "{{ default_container_dimensions }}"
 monasca_notification_dimensions: "{{ default_container_dimensions }}"
 monasca_persister_dimensions: "{{ default_container_dimensions }}"
 
-
 ####################
 # OpenStack
 ####################
@@ -201,6 +244,8 @@ monasca_default_authorized_roles:
   - admin
 monasca_read_only_authorized_roles:
   - monasca-read-only-user
+# NOTE(dszumski): The first role in this list is assigned to the monasca-agent
+# user for monitoring the OpenStack deployment.
 monasca_agent_authorized_roles:
   - agent
 monasca_delegate_authorized_roles:

ansible/roles/monasca/handlers/main.yml

@@ -172,3 +172,68 @@
     - config_json.changed | bool
       or monasca_persister_confs.changed | bool
       or monasca_persister_container.changed | bool
+
+- name: Restart monasca-agent-collector container
+  vars:
+    service_name: "monasca-agent-collector"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_agent_collector_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    pid_mode: "{{ service.pid_mode }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_agent_collector_confs.changed | bool
+      or monasca_agent_collector_plugin_confs.changed | bool
+      or monasca_agent_collector_container.changed | bool
+
+- name: Restart monasca-agent-forwarder container
+  vars:
+    service_name: "monasca-agent-forwarder"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_agent_forwarder_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_agent_forwarder_confs.changed | bool
+      or monasca_agent_forwarder_container.changed | bool
+
+- name: Restart monasca-agent-statsd container
+  vars:
+    service_name: "monasca-agent-statsd"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_agent_statsd_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_agent_statsd_confs.changed | bool
+      or monasca_agent_statsd_container.changed | bool

ansible/roles/monasca/tasks/config.yml

@@ -27,6 +27,107 @@
   notify:
     - "Restart {{ item.key }} container"
 
+- name: Copying over monasca-agent-collector config
+  vars:
+    service: "{{ monasca_services['monasca-agent-collector'] }}"
+  merge_yaml:
+    sources:
+      - "{{ role_path }}/templates/monasca-agent-collector/{{ item }}.j2"
+      - "{{ node_custom_config }}/monasca/{{ item }}"
+      - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-agent-collector/{{ item }}"
+    mode: "0660"
+  become: true
+  register: monasca_agent_collector_confs
+  with_items:
+    - agent-collector.yml
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-agent-collector container
+
+- name: Ensuring monasca-agent collector plugin config directory exists
+  vars:
+    service: "{{ monasca_services['monasca-agent-collector'] }}"
+  file:
+    path: "{{ node_config_directory }}/monasca-agent-collector/plugins"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+  become: true
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+
+- name: Find monasca-agent-collector plugin configuration files
+  find:
+    paths:
+      - "{{ role_path }}/templates/monasca-agent-collector/plugins/"
+      - "{{ node_custom_config }}/monasca/agent_plugins/"
+      - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/agent_plugins/"
+    patterns: '*.yaml'
+  run_once: True
+  register: agent_plugins
+
+- name: Copying over monasca-agent-collector plugins
+  vars:
+    service: "{{ monasca_services['monasca-agent-collector'] }}"
+  template:
+    src: "{{ item.path }}"
+    dest: "{{ node_config_directory }}/monasca-agent-collector/plugins/{{ item.path | basename }}"
+    mode: "0660"
+  become: true
+  register: monasca_agent_collector_plugin_confs
+  with_items:
+    "{{ agent_plugins.files }}"
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-agent-collector container
+
+- name: Copying over monasca-agent-forwarder config
+  vars:
+    service: "{{ monasca_services['monasca-agent-forwarder'] }}"
+  merge_yaml:
+    sources:
+      - "{{ role_path }}/templates/monasca-agent-forwarder/{{ item }}.j2"
+      - "{{ node_custom_config }}/monasca/{{ item }}"
+      - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-agent-forwarder/{{ item }}"
+    mode: "0660"
+  become: true
+  register: monasca_agent_forwarder_confs
+  with_items:
+    - agent-forwarder.yml
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-agent-forwarder container
+
+- name: Copying over monasca-agent-statsd config
+  vars:
+    service: "{{ monasca_services['monasca-agent-statsd'] }}"
+  merge_yaml:
+    sources:
+      - "{{ role_path }}/templates/monasca-agent-statsd/{{ item }}.j2"
+      - "{{ node_custom_config }}/monasca/{{ item }}"
+      - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-agent-statsd/{{ item }}"
+    mode: "0660"
+  become: true
+  register: monasca_agent_statsd_confs
+  with_items:
+    - agent-statsd.yml
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-agent-statsd container
+
 - name: Copying over monasca-api config
   vars:
     service: "{{ monasca_services['monasca-api'] }}"
@@ -263,6 +364,7 @@
     common_options: "{{ docker_common_options }}"
     name: "{{ item.value.container_name }}"
     image: "{{ item.value.image }}"
+    pid_mode: "{{ item.value.pid_mode|default('') }}"
     volumes: "{{ item.value.volumes }}"
     dimensions: "{{ item.value.dimensions }}"
   register: check_monasca_containers

ansible/roles/monasca/tasks/deploy.yml

@@ -1,10 +1,12 @@
 ---
 - include_tasks: register.yml
-  when: inventory_hostname in groups['monasca-api'] or
+  when: inventory_hostname in groups['monasca-agent'] or
+        inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api']
 
 - include_tasks: config.yml
-  when: inventory_hostname in groups['monasca-api'] or
+  when: inventory_hostname in groups['monasca-agent'] or
+        inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api'] or
         inventory_hostname in groups['monasca-log-transformer'] or
         inventory_hostname in groups['monasca-log-persister'] or
@@ -20,7 +22,8 @@
   meta: flush_handlers
 
 - include_tasks: check.yml
-  when: inventory_hostname in groups['monasca-api'] or
+  when: inventory_hostname in groups['monasca-agent'] or
+        inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api'] or
         inventory_hostname in groups['monasca-log-transformer'] or
         inventory_hostname in groups['monasca-log-persister'] or

ansible/roles/monasca/tasks/precheck.yml

@@ -25,3 +25,25 @@
   when:
     - inventory_hostname in groups[monasca_services['monasca-log-api']['group']]
     - container_facts['monasca_log_api'] is not defined
+
+- name: Checking free port for monasca-agent-forwarder
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ monasca_agent_forwarder_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - inventory_hostname in groups[monasca_services['monasca-agent-forwarder']['group']]
+    - container_facts['monasca_agent_forwarder'] is not defined
+
+- name: Checking free port for monasca-agent-statsd
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ monasca_agent_statsd_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - inventory_hostname in groups[monasca_services['monasca-agent-statsd']['group']]
+    - container_facts['monasca_agent_statsd'] is not defined

ansible/roles/monasca/tasks/register.yml

@@ -64,3 +64,15 @@
     - "{{ monasca_agent_authorized_roles }}"
     - "{{ monasca_read_only_authorized_roles }}"
     - "{{ monasca_delegate_authorized_roles }}"
+
+- name: Creating the monasca agent user
+  kolla_toolbox:
+    module_name: "kolla_keystone_user"
+    module_args:
+      project: "{{ monasca_control_plane_project }}"
+      user: "{{ monasca_agent_user }}"
+      password: "{{ monasca_agent_password }}"
+      role: "{{ monasca_agent_authorized_roles | first }}"
+      region_name: "{{ openstack_region_name }}"
+      auth: "{{ monasca_openstack_auth }}"
+  run_once: True

ansible/roles/monasca/templates/monasca-agent-collector/agent-collector.yml.j2

@@ -0,0 +1,9 @@
+Main:
+  hostname: {{ ansible_hostname }}
+  check_freq: {{ monasca_agent_check_frequency }}
+  forwarder_url: http://127.0.0.1:{{ monasca_agent_forwarder_port }}
+
+Logging:
+  log_level: {{ 'DEBUG' if monasca_logging_debug else 'INFO' }}
+  collector_log_file: /var/log/kolla/monasca/agent-collector.log
+  enable_logrotate: False

ansible/roles/monasca/templates/monasca-agent-collector/monasca-agent-collector.json.j2

@@ -0,0 +1,24 @@
+{
+    "command": "monasca-collector foreground --config-file /etc/monasca/agent-collector.yml",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/agent-collector.yml",
+            "dest": "/etc/monasca/agent-collector.yml",
+            "owner": "monasca",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/plugins/*.yaml",
+            "dest": "/etc/monasca/conf.d/",
+            "owner": "monasca",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/monasca",
+            "owner": "monasca:kolla",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/monasca/templates/monasca-agent-collector/plugins/cpu.yaml

@@ -0,0 +1,5 @@
+---
+init_config: null
+instances:
+  - built_by: System
+    name: cpu_stats

ansible/roles/monasca/templates/monasca-agent-collector/plugins/disk.yaml

@@ -0,0 +1,7 @@
+---
+init_config: null
+instances:
+  - built_by: System
+    device_blacklist_re: .*freezer_backup_snap.*
+    ignore_filesystem_types: iso9660,tmpfs
+    name: disk_stats

ansible/roles/monasca/templates/monasca-agent-collector/plugins/memory.yaml

@@ -0,0 +1,5 @@
+---
+init_config: null
+instances:
+  - built_by: System
+    name: memory_stats

ansible/roles/monasca/templates/monasca-agent-collector/plugins/network.yaml

@@ -0,0 +1,6 @@
+---
+init_config: null
+instances:
+  - built_by: System
+    excluded_interface_re: lo.*|vnet.*|tun.*|ovs.*|br.*|tap.*|qbr.*|qvb.*|qvo.*
+    name: network_stats

ansible/roles/monasca/templates/monasca-agent-forwarder/agent-forwarder.yml.j2

@@ -0,0 +1,26 @@
+Api:
+  service_type: monitoring
+  endpoint_type: internal
+  region_name: {{ openstack_region_name }}
+  username: {{ monasca_agent_user }}
+  password: {{ monasca_agent_password }}
+  keystone_url: {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}/v3
+  user_domain_name: Default
+  project_name: {{ monasca_control_plane_project }}
+  project_domain_id: {{ default_project_domain_id }}
+  project_domain_name: {{ default_project_domain_name }}
+  insecure: False
+  ca_file: /var/lib/kolla/venv/lib/python2.7/site-packages/certifi/cacert.pem
+  max_measurement_buffer_size: {{ monasca_agent_max_buffer_size }}
+  backlog_send_rate: {{ monasca_agent_backlog_send_rate }}
+  max_batch_size: {{ monasca_agent_max_batch_size }}
+
+Main:
+  hostname: {{ ansible_hostname }}
+  non_local_traffic: True
+  listen_port: {{ monasca_agent_forwarder_port }}
+
+Logging:
+  log_level: {{ 'DEBUG' if monasca_logging_debug else 'INFO' }}
+  forwarder_log_file: /var/log/kolla/monasca/agent-forwarder.log
+  enable_logrotate: False

ansible/roles/monasca/templates/monasca-agent-forwarder/monasca-agent-forwarder.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "monasca-forwarder --config-file=/etc/monasca/agent-forwarder.yml",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/agent-forwarder.yml",
+            "dest": "/etc/monasca/agent-forwarder.yml",
+            "owner": "monasca",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/monasca",
+            "owner": "monasca:kolla",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/monasca/templates/monasca-agent-statsd/agent-statsd.yml.j2

@@ -0,0 +1,12 @@
+Main:
+  hostname: {{ ansible_hostname }}
+  forwarder_url: http://127.0.0.1:{{ monasca_agent_forwarder_port }}
+
+Statsd:
+  monasca_statsd_port : {{ monasca_agent_statsd_port }}
+  non_local_traffic: True
+
+Logging:
+  log_level: {{ 'DEBUG' if monasca_logging_debug else 'INFO' }}
+  statsd_log_file: /var/log/kolla/monasca/agent-statsd.log
+  enable_logrotate: False

ansible/roles/monasca/templates/monasca-agent-statsd/monasca-agent-statsd.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "monasca-statsd --config-file /etc/monasca/agent-statsd.yml",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/agent-statsd.yml",
+            "dest": "/etc/monasca/agent-statsd.yml",
+            "owner": "monasca",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/monasca",
+            "owner": "monasca:kolla",
+            "recurse": true
+        }
+    ]
+}

ansible/site.yml

@@ -812,7 +812,9 @@
 
 - name: Apply role monasca
   gather_facts: false
-  hosts: monasca
+  hosts:
+    - monasca
+    - monasca-agent
   serial: '{{ kolla_serial|default("0") }}'
   roles:
     - { role: monasca,

etc/kolla/passwords.yml

@@ -128,6 +128,7 @@ murano_database_password:
 murano_keystone_password:
 murano_agent_rabbitmq_password:
 
+monasca_agent_password:
 monasca_database_password:
 monasca_keystone_password:
 

releasenotes/notes/add_monasca_agent-2b3b2f484a9e7168.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Add the Monasca Agent which provides host and application
+    specific monitoring data collection and forwarding.