Support deploying the Monasca Agent

The Monasca Agent collects metrics and in this change is deployed
across the control plane. These metrics are collected into an OpenStack
project. It supports configuring a small number of plugins, which can
be extended in later commits. It also makes the Monasca Agent credentials
available to other roles, such as the common role to allow forwarding
logs to Monasca.

Partially-Implements: blueprint monasca-roles
Change-Id: I76b34fc5e1c76407a45fcf272268d5798b473ca2

ansible/group_vars/all.yml

@@ -259,6 +259,8 @@ mistral_api_port: "8989"
 
 monasca_api_port: "8070"
 monasca_log_api_port: "5607"
+monasca_agent_forwarder_port: "17123"
+monasca_agent_statsd_port: "8125"
 
 mongodb_port: "27017"
 mongodb_web_port: "28017"
@@ -564,6 +566,19 @@ enable_prometheus: "no"
 # Clean images options are specified here
 enable_destroy_images: "no"
 
+####################
+# Monasca options
+####################
+
+# The OpenStack username used by the Monasca Agent and the Fluentd Monasca
+# plugin to post logs and metrics from the control plane to Monasca.
+monasca_agent_user: "monasca-agent"
+
+# The OpenStack project to which the control plane logs and metrics are
+# tagged with. Only users with the monasca read only user role, or higher
+# can access these from the Monasca APIs.
+monasca_control_plane_project: "monasca_control_plane"
+
 ####################
 # Logging options
 ####################

ansible/inventory/all-in-one

@@ -99,6 +99,13 @@ control
 [qdrouterd:children]
 control
 
+[monasca-agent:children]
+compute
+control
+monitoring
+network
+storage
+
 [monasca:children]
 monitoring
 
@@ -445,6 +452,15 @@ murano
 murano
 
 # Monasca
+[monasca-agent-collector:children]
+monasca-agent
+
+[monasca-agent-forwarder:children]
+monasca-agent
+
+[monasca-agent-statsd:children]
+monasca-agent
+
 [monasca-api:children]
 monasca
 

ansible/inventory/multinode

@@ -124,6 +124,13 @@ control
 [qdrouterd:children]
 control
 
+[monasca-agent:children]
+compute
+control
+monitoring
+network
+storage
+
 [monasca:children]
 monitoring
 
@@ -454,6 +461,15 @@ murano
 murano
 
 # Monasca
+[monasca-agent-collector:children]
+monasca-agent
+
+[monasca-agent-forwarder:children]
+monasca-agent
+
+[monasca-agent-statsd:children]
+monasca-agent
+
 [monasca-api:children]
 monasca
 

ansible/roles/monasca/defaults/main.yml

@@ -103,6 +103,39 @@ monasca_services:
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla"
     dimensions: "{{ monasca_persister_dimensions }}"
+  monasca-agent-collector:
+    container_name: monasca_agent_collector
+    group: monasca-agent-collector
+    enabled: true
+    image: "{{ monasca_agent_image_full }}"
+    pid_mode: "host"
+    volumes:
+      - "{{ node_config_directory }}/monasca-agent-collector/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
+      - "/sys:/sys:ro"
+      - "/dev/disk/:/dev/disk:ro"
+    dimensions: "{{ monasca_agent_dimensions }}"
+  monasca-agent-statsd:
+    container_name: monasca_agent_statsd
+    group: monasca-agent-statsd
+    enabled: true
+    image: "{{ monasca_agent_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/monasca-agent-statsd/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
+    dimensions: "{{ monasca_agent_dimensions }}"
+  monasca-agent-forwarder:
+    container_name: monasca_agent_forwarder
+    group: monasca-agent-forwarder
+    enabled: true
+    image: "{{ monasca_agent_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/monasca-agent-forwarder/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
+    dimensions: "{{ monasca_agent_dimensions }}"
 
 ####################
 # Databases
@@ -142,6 +175,12 @@ monasca_periodic_notifications_topic: "60-seconds-notifications"
 # implemented it is not recommended to change this period.
 monasca_periodic_notifications_period: 60
 
+# Agent settings
+monasca_agent_max_buffer_size: 1000
+monasca_agent_backlog_send_rate: 1000
+monasca_agent_max_batch_size: 1000
+monasca_agent_check_frequency: 30
+
 # Processing pipeline threads. In a large scale deployment you will likely
 # want to tune these with finer precision. For example, if you have a very
 # high log throughput, the log metrics service consumer may require a
@@ -157,6 +196,10 @@ monasca_metric_pipeline_threads: 2
 monasca_install_type: "{{ kolla_install_type }}"
 monasca_tag: "{{ openstack_release }}"
 
+monasca_agent_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-agent"
+monasca_agent_tag: "{{ monasca_tag }}"
+monasca_agent_image_full: "{{ monasca_agent_image }}:{{ monasca_agent_tag }}"
+
 monasca_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-api"
 monasca_api_tag: "{{ monasca_tag }}"
 monasca_api_image_full: "{{ monasca_api_image }}:{{ monasca_api_tag }}"
@@ -181,6 +224,7 @@ monasca_persister_image: "{{ docker_registry ~ '/' if docker_registry else '' }}
 monasca_persister_tag: "{{ monasca_tag }}"
 monasca_persister_image_full: "{{ monasca_persister_image }}:{{ monasca_persister_tag }}"
 
+monasca_agent_dimensions: "{{ default_container_dimensions }}"
 monasca_api_dimensions: "{{ default_container_dimensions }}"
 monasca_log_api_dimensions: "{{ default_container_dimensions }}"
 monasca_log_transformer_dimensions: "{{ default_container_dimensions }}"
@@ -190,7 +234,6 @@ monasca_thresh_dimensions: "{{ default_container_dimensions }}"
 monasca_notification_dimensions: "{{ default_container_dimensions }}"
 monasca_persister_dimensions: "{{ default_container_dimensions }}"
 
-
 ####################
 # OpenStack
 ####################
@@ -201,6 +244,8 @@ monasca_default_authorized_roles:
   - admin
 monasca_read_only_authorized_roles:
   - monasca-read-only-user
+# NOTE(dszumski): The first role in this list is assigned to the monasca-agent
+# user for monitoring the OpenStack deployment.
 monasca_agent_authorized_roles:
   - agent
 monasca_delegate_authorized_roles:

ansible/roles/monasca/handlers/main.yml

@@ -172,3 +172,68 @@
     - config_json.changed | bool
       or monasca_persister_confs.changed | bool
       or monasca_persister_container.changed | bool
+
+- name: Restart monasca-agent-collector container
+  vars:
+    service_name: "monasca-agent-collector"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_agent_collector_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    pid_mode: "{{ service.pid_mode }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_agent_collector_confs.changed | bool
+      or monasca_agent_collector_plugin_confs.changed | bool
+      or monasca_agent_collector_container.changed | bool
+
+- name: Restart monasca-agent-forwarder container
+  vars:
+    service_name: "monasca-agent-forwarder"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_agent_forwarder_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_agent_forwarder_confs.changed | bool
+      or monasca_agent_forwarder_container.changed | bool
+
+- name: Restart monasca-agent-statsd container
+  vars:
+    service_name: "monasca-agent-statsd"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_agent_statsd_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_agent_statsd_confs.changed | bool
+      or monasca_agent_statsd_container.changed | bool

ansible/roles/monasca/tasks/config.yml

@@ -27,6 +27,107 @@
   notify:
     - "Restart {{ item.key }} container"
 
+- name: Copying over monasca-agent-collector config
+  vars:
+    service: "{{ monasca_services['monasca-agent-collector'] }}"
+  merge_yaml:
+    sources:
+      - "{{ role_path }}/templates/monasca-agent-collector/{{ item }}.j2"
+      - "{{ node_custom_config }}/monasca/{{ item }}"
+      - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-agent-collector/{{ item }}"
+    mode: "0660"
+  become: true
+  register: monasca_agent_collector_confs
+  with_items:
+    - agent-collector.yml
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-agent-collector container
+
+- name: Ensuring monasca-agent collector plugin config directory exists
+  vars:
+    service: "{{ monasca_services['monasca-agent-collector'] }}"
+  file:
+    path: "{{ node_config_directory }}/monasca-agent-collector/plugins"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+  become: true
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+
+- name: Find monasca-agent-collector plugin configuration files
+  find:
+    paths:
+      - "{{ role_path }}/templates/monasca-agent-collector/plugins/"
+      - "{{ node_custom_config }}/monasca/agent_plugins/"
+      - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/agent_plugins/"
+    patterns: '*.yaml'
+  run_once: True
+  register: agent_plugins
+
+- name: Copying over monasca-agent-collector plugins
+  vars:
+    service: "{{ monasca_services['monasca-agent-collector'] }}"
+  template:
+    src: "{{ item.path }}"
+    dest: "{{ node_config_directory }}/monasca-agent-collector/plugins/{{ item.path | basename }}"
+    mode: "0660"
+  become: true
+  register: monasca_agent_collector_plugin_confs
+  with_items:
+    "{{ agent_plugins.files }}"
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-agent-collector container
+
+- name: Copying over monasca-agent-forwarder config
+  vars:
+    service: "{{ monasca_services['monasca-agent-forwarder'] }}"
+  merge_yaml:
+    sources:
+      - "{{ role_path }}/templates/monasca-agent-forwarder/{{ item }}.j2"
+      - "{{ node_custom_config }}/monasca/{{ item }}"
+      - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-agent-forwarder/{{ item }}"
+    mode: "0660"
+  become: true
+  register: monasca_agent_forwarder_confs
+  with_items:
+    - agent-forwarder.yml
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-agent-forwarder container
+
+- name: Copying over monasca-agent-statsd config
+  vars:
+    service: "{{ monasca_services['monasca-agent-statsd'] }}"
+  merge_yaml:
+    sources:
+      - "{{ role_path }}/templates/monasca-agent-statsd/{{ item }}.j2"
+      - "{{ node_custom_config }}/monasca/{{ item }}"
+      - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-agent-statsd/{{ item }}"
+    mode: "0660"
+  become: true
+  register: monasca_agent_statsd_confs
+  with_items:
+    - agent-statsd.yml
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-agent-statsd container
+
 - name: Copying over monasca-api config
   vars:
     service: "{{ monasca_services['monasca-api'] }}"
@@ -263,6 +364,7 @@
     common_options: "{{ docker_common_options }}"
     name: "{{ item.value.container_name }}"
     image: "{{ item.value.image }}"
+    pid_mode: "{{ item.value.pid_mode|default('') }}"
     volumes: "{{ item.value.volumes }}"
     dimensions: "{{ item.value.dimensions }}"
   register: check_monasca_containers

ansible/roles/monasca/tasks/deploy.yml

@@ -1,10 +1,12 @@
 ---
 - include_tasks: register.yml
-  when: inventory_hostname in groups['monasca-api'] or
+  when: inventory_hostname in groups['monasca-agent'] or
+        inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api']
 
 - include_tasks: config.yml
-  when: inventory_hostname in groups['monasca-api'] or
+  when: inventory_hostname in groups['monasca-agent'] or
+        inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api'] or
         inventory_hostname in groups['monasca-log-transformer'] or
         inventory_hostname in groups['monasca-log-persister'] or
@@ -20,7 +22,8 @@
   meta: flush_handlers
 
 - include_tasks: check.yml
-  when: inventory_hostname in groups['monasca-api'] or
+  when: inventory_hostname in groups['monasca-agent'] or
+        inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api'] or
         inventory_hostname in groups['monasca-log-transformer'] or
         inventory_hostname in groups['monasca-log-persister'] or

ansible/roles/monasca/tasks/precheck.yml

@@ -25,3 +25,25 @@
   when:
     - inventory_hostname in groups[monasca_services['monasca-log-api']['group']]
     - container_facts['monasca_log_api'] is not defined
+
+- name: Checking free port for monasca-agent-forwarder
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ monasca_agent_forwarder_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - inventory_hostname in groups[monasca_services['monasca-agent-forwarder']['group']]
+    - container_facts['monasca_agent_forwarder'] is not defined
+
+- name: Checking free port for monasca-agent-statsd
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ monasca_agent_statsd_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - inventory_hostname in groups[monasca_services['monasca-agent-statsd']['group']]
+    - container_facts['monasca_agent_statsd'] is not defined

ansible/roles/monasca/tasks/register.yml

@@ -64,3 +64,15 @@
     - "{{ monasca_agent_authorized_roles }}"
     - "{{ monasca_read_only_authorized_roles }}"
     - "{{ monasca_delegate_authorized_roles }}"
+
+- name: Creating the monasca agent user
+  kolla_toolbox:
+    module_name: "kolla_keystone_user"
+    module_args:
+      project: "{{ monasca_control_plane_project }}"
+      user: "{{ monasca_agent_user }}"
+      password: "{{ monasca_agent_password }}"
+      role: "{{ monasca_agent_authorized_roles | first }}"
+      region_name: "{{ openstack_region_name }}"
+      auth: "{{ monasca_openstack_auth }}"
+  run_once: True

ansible/roles/monasca/templates/monasca-agent-collector/agent-collector.yml.j2

@@ -0,0 +1,9 @@
+Main:
+  hostname: {{ ansible_hostname }}
+  check_freq: {{ monasca_agent_check_frequency }}
+  forwarder_url: http://127.0.0.1:{{ monasca_agent_forwarder_port }}
+
+Logging:
+  log_level: {{ 'DEBUG' if monasca_logging_debug else 'INFO' }}
+  collector_log_file: /var/log/kolla/monasca/agent-collector.log
+  enable_logrotate: False

ansible/roles/monasca/templates/monasca-agent-collector/monasca-agent-collector.json.j2

@@ -0,0 +1,24 @@
+{
+    "command": "monasca-collector foreground --config-file /etc/monasca/agent-collector.yml",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/agent-collector.yml",
+            "dest": "/etc/monasca/agent-collector.yml",
+            "owner": "monasca",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/plugins/*.yaml",
+            "dest": "/etc/monasca/conf.d/",
+            "owner": "monasca",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/monasca",
+            "owner": "monasca:kolla",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/monasca/templates/monasca-agent-collector/plugins/cpu.yaml

@@ -0,0 +1,5 @@
+---
+init_config: null
+instances:
+  - built_by: System
+    name: cpu_stats

ansible/roles/monasca/templates/monasca-agent-collector/plugins/disk.yaml

@@ -0,0 +1,7 @@
+---
+init_config: null
+instances:
+  - built_by: System
+    device_blacklist_re: .*freezer_backup_snap.*
+    ignore_filesystem_types: iso9660,tmpfs
+    name: disk_stats

ansible/roles/monasca/templates/monasca-agent-collector/plugins/memory.yaml

@@ -0,0 +1,5 @@
+---
+init_config: null
+instances:
+  - built_by: System
+    name: memory_stats

ansible/roles/monasca/templates/monasca-agent-collector/plugins/network.yaml

@@ -0,0 +1,6 @@
+---
+init_config: null
+instances:
+  - built_by: System
+    excluded_interface_re: lo.*|vnet.*|tun.*|ovs.*|br.*|tap.*|qbr.*|qvb.*|qvo.*
+    name: network_stats

ansible/roles/monasca/templates/monasca-agent-forwarder/agent-forwarder.yml.j2

@@ -0,0 +1,26 @@
+Api:
+  service_type: monitoring
+  endpoint_type: internal
+  region_name: {{ openstack_region_name }}
+  username: {{ monasca_agent_user }}
+  password: {{ monasca_agent_password }}
+  keystone_url: {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}/v3
+  user_domain_name: Default
+  project_name: {{ monasca_control_plane_project }}
+  project_domain_id: {{ default_project_domain_id }}
+  project_domain_name: {{ default_project_domain_name }}
+  insecure: False
+  ca_file: /var/lib/kolla/venv/lib/python2.7/site-packages/certifi/cacert.pem
+  max_measurement_buffer_size: {{ monasca_agent_max_buffer_size }}
+  backlog_send_rate: {{ monasca_agent_backlog_send_rate }}
+  max_batch_size: {{ monasca_agent_max_batch_size }}
+
+Main:
+  hostname: {{ ansible_hostname }}
+  non_local_traffic: True
+  listen_port: {{ monasca_agent_forwarder_port }}
+
+Logging:
+  log_level: {{ 'DEBUG' if monasca_logging_debug else 'INFO' }}
+  forwarder_log_file: /var/log/kolla/monasca/agent-forwarder.log
+  enable_logrotate: False

ansible/roles/monasca/templates/monasca-agent-forwarder/monasca-agent-forwarder.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "monasca-forwarder --config-file=/etc/monasca/agent-forwarder.yml",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/agent-forwarder.yml",
+            "dest": "/etc/monasca/agent-forwarder.yml",
+            "owner": "monasca",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/monasca",
+            "owner": "monasca:kolla",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/monasca/templates/monasca-agent-statsd/agent-statsd.yml.j2

@@ -0,0 +1,12 @@
+Main:
+  hostname: {{ ansible_hostname }}
+  forwarder_url: http://127.0.0.1:{{ monasca_agent_forwarder_port }}
+
+Statsd:
+  monasca_statsd_port : {{ monasca_agent_statsd_port }}
+  non_local_traffic: True
+
+Logging:
+  log_level: {{ 'DEBUG' if monasca_logging_debug else 'INFO' }}
+  statsd_log_file: /var/log/kolla/monasca/agent-statsd.log
+  enable_logrotate: False

ansible/roles/monasca/templates/monasca-agent-statsd/monasca-agent-statsd.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "monasca-statsd --config-file /etc/monasca/agent-statsd.yml",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/agent-statsd.yml",
+            "dest": "/etc/monasca/agent-statsd.yml",
+            "owner": "monasca",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/monasca",
+            "owner": "monasca:kolla",
+            "recurse": true
+        }
+    ]
+}

ansible/site.yml

@@ -812,7 +812,9 @@
 
 - name: Apply role monasca
   gather_facts: false
-  hosts: monasca
+  hosts:
+    - monasca
+    - monasca-agent
   serial: '{{ kolla_serial|default("0") }}'
   roles:
     - { role: monasca,

etc/kolla/passwords.yml

@@ -128,6 +128,7 @@ murano_database_password:
 murano_keystone_password:
 murano_agent_rabbitmq_password:
 
+monasca_agent_password:
 monasca_database_password:
 monasca_keystone_password:
 

releasenotes/notes/add_monasca_agent-2b3b2f484a9e7168.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Add the Monasca Agent which provides host and application
+    specific monitoring data collection and forwarding.