Add skydive service

This patch introduces the ansible materials to deploy
the skydive service, that can be used to monitor and
troubleshoot networking in an openstack deployment.

Implements: blueprint skydive-service

Co-Authored-By: Nicolas Bouron <nicolas.bouron@gmail.com>
Signed-off-by: Mathieu Rohon <mathieu.rohon@gmail.com>

Change-Id: I53051a1b0c85380416288e17040a398b6efb62c0

ansible/group_vars/all.yml

@@ -249,6 +249,10 @@ zun_api_port: "9517"
 
 ovsdb_port: "6640"
 
+skydive_analyzer_port: "8085"
+skydive_agents_port: "8090"
+
+
 public_protocol: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
 internal_protocol: "http"
 admin_protocol: "http"
@@ -362,6 +366,7 @@ enable_rally: "no"
 enable_sahara: "no"
 enable_searchlight: "no"
 enable_senlin: "no"
+enable_skydive: "no"
 enable_solum: "no"
 enable_swift: "no"
 enable_tacker: "no"
@@ -394,7 +399,7 @@ enable_destroy_images: "no"
 elasticsearch_address: "{{ kolla_internal_vip_address }}"
 elasticsearch_protocol: "{{ internal_protocol }}"
 
-enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_freezer | bool or enable_osprofiler | bool else 'no' }}"
+enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_freezer | bool or enable_osprofiler | bool or enable_skydive | bool else 'no' }}"
 enable_kibana: "{{ 'yes' if enable_central_logging | bool else 'no' }}"
 
 ####################

ansible/inventory/all-in-one

@@ -192,6 +192,9 @@ deployment
 [zun:children]
 control
 
+[skydive:children]
+monitoring
+
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
 #
@@ -542,3 +545,12 @@ zun
 
 [zun-compute:children]
 compute
+
+# Skydive
+[skydive-analyzer:children]
+skydive
+
+[skydive-agent:children]
+compute
+network
+

ansible/inventory/multinode

@@ -210,6 +210,9 @@ deployment
 [zun:children]
 control
 
+[skydive:children]
+monitoring
+
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
 #
@@ -560,3 +563,11 @@ zun
 
 [zun-compute:children]
 compute
+
+# Skydive
+[skydive-analyzer:children]
+skydive
+
+[skydive-agent:children]
+compute
+network

ansible/roles/haproxy/templates/haproxy.cfg.j2

@@ -822,6 +822,22 @@ listen zun_api_external
 {% endif %}
 {% endif %}
 
+{% if enable_skydive | bool %}
+listen skydive_server
+  bind {{ kolla_internal_vip_address }}:{{ skydive_analyzer_port }}
+{% for host in groups['skydive-analyzer'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ skydive_analyzer_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% if haproxy_enable_external_vip | bool %}
+
+listen skydive_server_external
+  bind {{ kolla_external_vip_address }}:{{ skydive_analyzer_port }} {{ tls_bind_info }}
+{% for host in groups['skydive-analyzer'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ skydive_analyzer_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% endif %}
+{% endif %}
+
 # (NOTE): This defaults section deletes forwardfor as recommended by:
 #         https://marc.info/?l=haproxy&m=141684110710132&w=1
 
@@ -852,3 +868,4 @@ listen mariadb
 
 {% endfor %}
 {% endif %}
+

ansible/roles/skydive/defaults/main.yml

@@ -0,0 +1,35 @@
+---
+project_name: "skydive"
+
+skydive_services:
+  skydive-analyzer:
+    container_name: skydive_analyzer
+    group: skydive-analyzer
+    enabled: true
+    image: "{{ skydive_analyzer_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/skydive-analyzer/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+  skydive-agent:
+    container_name: skydive_agent
+    group: skydive-agent
+    enabled: true
+    image: "{{ skydive_agent_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/skydive-agent/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "/var/run/openvswitch/db.sock:/var/run/openvswitch/db.sock:ro"
+      - "kolla_logs:/var/log/kolla/"
+
+####################
+# Docker
+####################
+skydive_analyzer_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-skydive-analyzer"
+skydive_analyzer_tag: "{{ openstack_release }}"
+skydive_analyzer_image_full: "{{ skydive_analyzer_image }}:{{ skydive_analyzer_tag }}"
+
+skydive_agent_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-skydive-agent"
+skydive_agent_tag: "{{ openstack_release }}"
+skydive_agent_image_full: "{{ skydive_agent_image }}:{{ skydive_agent_tag }}"
+

ansible/roles/skydive/handlers/main.yml

@@ -0,0 +1,43 @@
+---
+- name: Restart skydive-analyzer container
+  vars:
+    service_name: "skydive-analyzer"
+    service: "{{ skydive_services[service_name] }}"
+    config_json: "{{ skydive_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    skydive_conf: "{{ skydive_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    skydive_analyzer_container: "{{ check_skydive_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+  when:
+    - action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or skydive_conf.changed |bool
+      or skydive_analyzer_container.changed | bool
+
+- name: Restart skydive-agent container
+  vars:
+    service_name: "skydive-agent"
+    service: "{{ skydive_services[service_name] }}"
+    config_json: "{{ skydive_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    skydive_conf: "{{ skydive_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    skydive_agent_container: "{{ check_skydive_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+  when:
+    - action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or skydive_conf.changed |bool
+      or skydive_agent_container.changed | bool
+

ansible/roles/skydive/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: common }

ansible/roles/skydive/tasks/config.yml

@@ -0,0 +1,53 @@
+---
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item.key }}/config"
+    state: "directory"
+    recurse: yes
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ skydive_services }}"
+
+- name: Copying over default config.json files
+  template:
+    src: "{{ item.key }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
+  register: skydive_config_jsons
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ skydive_services }}"
+  notify:
+    - Restart skydive-analyzer container
+    - Restart skydive-agent container
+
+- name: Copying over skydive config file
+  template:
+    src: "{{ item.key }}.conf.j2"
+    dest: "{{ node_config_directory }}/{{ item.key }}/skydive.conf"
+  register: skydive_confs
+  when:
+    - item.value.enabled | bool
+    - inventory_hostname in groups[item.value.group]
+  with_dict: "{{ skydive_services }}"
+  notify:
+    - Restart skydive-analyzer container
+    - Restart skydive-agent container
+
+- name: Check skydive containers
+  kolla_docker:
+    action: "compare_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ item.value.container_name }}"
+    image: "{{ item.value.image }}"
+    volumes: "{{ item.value.volumes }}"
+  register: check_skydive_containers
+  when:
+    - action != "config"
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ skydive_services }}"
+  notify:
+    - Restart skydive-analyzer container
+    - Restart skydive-agent container

ansible/roles/skydive/tasks/deploy.yml

@@ -0,0 +1,7 @@
+---
+- include: config.yml
+  when: inventory_hostname in groups['skydive-analyzer'] or
+        inventory_hostname in groups['skydive-agent']
+
+- name: Flush handlers
+  meta: flush_handlers

ansible/roles/skydive/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include: "{{ action }}.yml"

ansible/roles/skydive/tasks/precheck.yml

@@ -0,0 +1,34 @@
+---
+- name: Get container facts
+  kolla_container_facts:
+    name:
+      - skydive_analyzer
+      - skydive_agent
+  register: container_facts
+
+- name: Checking free port for Skydive Analyzer
+  vars:
+    skydive_analyzer: "{{ skydive_services['skydive-analyzer'] }}"
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ skydive_analyzer_port }}"
+    connect_timeout: 1
+    state: stopped
+  when:
+    - container_facts['skydive_analyzer'] is not defined
+    - inventory_hostname in groups[skydive_analyzer.group]
+    - skydive_analyzer.enabled | bool
+
+- name: Checking free port for Skydive Agent
+  vars:
+    skydive_agent: "{{ skydive_services['skydive-agent'] }}"
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ skydive_agents_port }}"
+    connect_timeout: 1
+    state: stopped
+  when:
+    - container_facts['skydive_agent'] is not defined
+    - inventory_hostname in groups[skydive_agent.group]
+    - skydive_agent.enabled | bool
+

ansible/roles/skydive/tasks/pull.yml

@@ -0,0 +1,10 @@
+---
+- name: Pulling skydive images
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ item.value.image }}"
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ skydive_services }}"

ansible/roles/skydive/tasks/reconfigure.yml

@@ -0,0 +1,2 @@
+---
+- include: "deploy.yml"

ansible/roles/skydive/tasks/upgrade.yml

@@ -0,0 +1,5 @@
+---
+- include: config.yml
+
+- name: Flush handlers
+  meta: flush_handlers

ansible/roles/skydive/templates/skydive-agent.conf.j2

@@ -0,0 +1,54 @@
+### Skydive agent config file
+
+auth:
+  type: keystone
+  analyzer_username: {{ openstack_auth['username'] }}
+  analyzer_password: {{ openstack_auth['password'] }}
+
+logging:
+  default: DEBUG
+  topology/probes: INFO
+  topology/graph: INFO
+
+openstack:
+  auth_url: {{ keystone_public_url }}/v3
+  username: {{ openstack_auth['username'] }}
+  password: {{ openstack_auth['password'] }}
+  tenant_name: {{ openstack_auth['project_name'] }}
+  region_name: {{ openstack_region_name }}
+  domain_name: Default
+
+etcd:
+  servers:
+{% if enable_etcd == "yes" %}
+{% for host in groups['etcd'] %}
+    - http://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ etcd_client_port }}
+{% endfor %}
+{% else %}
+{% for host in groups['skydive-analyzer'] %}
+  - http://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ etcd_client_port }}
+{% endfor %}
+{% endif %}
+
+analyzers:
+{% for host in groups['skydive-analyzer'] %}
+  - {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ skydive_analyzer_port }}
+{% endfor %}
+
+agent:
+  listen: {{ hostvars[inventory_hostname]['ansible_' + hostvars[inventory_hostname]['api_interface']]['ipv4']['address'] }}:{{ skydive_agents_port }}
+  flow:
+    probes:
+      - gopacket
+{% if neutron_plugin_agent == "openvswitch" %}
+      - ovssflow
+{% endif %}
+  topology:
+    probes:
+      - netlink
+      - netns
+      - neutron
+{% if neutron_plugin_agent == "openvswitch" %}
+      - ovsdb
+{% endif %}
+

ansible/roles/skydive/templates/skydive-agent.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "skydive agent --conf /etc/skydive/skydive.conf --listen={{ api_interface_address }}:{{ skydive_agents_port }}",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/skydive.conf",
+            "dest": "/etc/skydive/skydive.conf",
+            "owner": "skydive",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/skydive",
+            "owner": "skydive:skydive",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/skydive/templates/skydive-analyzer.conf.j2

@@ -0,0 +1,62 @@
+### Skydive analyzer config file
+
+auth:
+  type: keystone
+
+logging:
+  default: DEBUG
+  topology/probes: INFO
+  topology/graph: INFO
+
+openstack:
+  auth_url: {{ keystone_public_url }}/v3
+  tenant_name: {{ openstack_auth['project_name'] }}
+  region_name: {{ openstack_region_name }}
+  domain_name: Default
+
+etcd:
+  client_timeout: 100
+{% if enable_etcd == "yes" %}
+  embeded: no
+  servers:
+{% for host in groups['etcd'] %}
+    - http://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ etcd_client_port }}
+{% endfor %}
+{% else %}
+  embeded: yes
+  servers:
+{% for host in groups['skydive-analyzer'] %}
+  - http://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ etcd_client_port }}
+{% endfor %}
+  listen: {{ api_interface_address }}:{{ etcd_client_port }}
+{% endif %}
+
+analyzer:
+  listen: {{ api_interface_address }}:{{ skydive_analyzer_port }}
+  flowtable_expire: 60
+  flowtable_update: 5
+  flowtable_agent_ratio: 0.5
+  storage:
+    backend: elasticsearch
+{% if groups['skydive-agent'] | length > 1 %}
+  topology:
+    fabric:
+{% for interface in [network_interface, neutron_external_interface]|unique %}
+{% set interfaces_loop = loop %}
+{% for host in groups['skydive-agent'] %}
+      - TOR{{ interfaces_loop.index }}[Name=tor{{ interfaces_loop.index }}] -> TOR{{ interfaces_loop.index }}_PORT{{ loop.index }}[Name=port{{ loop.index }}, MTU=1500]
+      - TOR{{ interfaces_loop.index }}_PORT{{ loop.index }} -> *[Type=host,Name={{ hostvars[host]['ansible_hostname'] }}]/{{ interface }}
+{% endfor %}
+{% endfor %}
+{% endif %}
+
+storage:
+  elasticsearch:
+    host: {{ elasticsearch_address }}:{{ elasticsearch_port }}
+    maxconns: 10
+    retry: 60
+
+graph:
+  embeded: true
+  client_timeout: 100
+

ansible/roles/skydive/templates/skydive-analyzer.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "skydive analyzer --conf /etc/skydive/skydive.conf --listen={{ api_interface_address }}:{{ skydive_analyzer_port }}",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/skydive.conf",
+            "dest": "/etc/skydive/skydive.conf",
+            "owner": "skydive",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/skydive",
+            "owner": "skydive:skydive",
+            "recurse": true
+        }
+    ]
+}

ansible/site.yml

@@ -634,3 +634,14 @@
     - { role: zun,
         tags: zun,
         when: enable_zun | bool }
+
+- name: Apply role skydive
+  hosts:
+    - skydive-agent
+    - skydive-analyzer
+  serial: '{{ serial|default("0") }}'
+  roles:
+    - { role: skydive,
+        tags: skydive,
+        when: enable_skydive | bool }
+

etc/kolla/globals.yml

@@ -186,6 +186,7 @@ kolla_internal_vip_address: "10.10.10.254"
 #enable_sahara: "no"
 #enable_searchlight: "no"
 #enable_senlin: "no"
+#enable_skydive: "no"
 #enable_solum: "no"
 #enable_swift: "no"
 #enable_telegraf: "no"