From 140722f74e63015240a1d6add50bc4603de57856 Mon Sep 17 00:00:00 2001 From: Dawud Date: Mon, 20 Nov 2023 15:47:27 +0000 Subject: [PATCH] Enable HAProxy Prometheus metrics endpoint HAProxy exposes a Prometheus metrics endpoint, it just needs to be enabled. Enable this and remove configuration for prometheus-haproxy-exporter. Remaining prometheus-haproxy-exporter containers will automatically be removed. Change-Id: If6e75691d2a996b06a9b95cb0aae772db54389fb Co-Authored-By: Matt Anson --- ansible/inventory/all-in-one | 3 --- ansible/inventory/multinode | 3 --- ansible/roles/loadbalancer/tasks/upgrade.yml | 18 +++++++++++++++++ .../templates/haproxy/haproxy_main.cfg.j2 | 7 +++++++ ansible/roles/prometheus/defaults/main.yml | 20 ------------------- ansible/roles/prometheus/handlers/main.yml | 15 -------------- ansible/roles/prometheus/tasks/precheck.yml | 13 ------------ .../prometheus-haproxy-exporter.json.j2 | 11 ---------- .../prometheus/templates/prometheus.yml.j2 | 4 ++-- ansible/site.yml | 1 - etc/kolla/globals.yml | 1 - ...oxy-metrics-endpoint-d62c0f34e29a10b2.yaml | 19 ++++++++++++++++++ specs/prometheus.rst | 6 +++--- tests/templates/inventory.j2 | 3 --- 14 files changed, 49 insertions(+), 75 deletions(-) delete mode 100644 ansible/roles/prometheus/templates/prometheus-haproxy-exporter.json.j2 create mode 100644 releasenotes/notes/haproxy-metrics-endpoint-d62c0f34e29a10b2.yaml diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 138022e73b..24ddc17604 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -620,9 +620,6 @@ storage [prometheus-mysqld-exporter:children] mariadb -[prometheus-haproxy-exporter:children] -loadbalancer - [prometheus-memcached-exporter:children] memcached diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index eaa077c37d..758879c0cb 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -639,9 +639,6 @@ storage [prometheus-mysqld-exporter:children] mariadb -[prometheus-haproxy-exporter:children] -loadbalancer - [prometheus-memcached-exporter:children] memcached diff --git a/ansible/roles/loadbalancer/tasks/upgrade.yml b/ansible/roles/loadbalancer/tasks/upgrade.yml index 5b10a7e111..5407bfed60 100644 --- a/ansible/roles/loadbalancer/tasks/upgrade.yml +++ b/ansible/roles/loadbalancer/tasks/upgrade.yml @@ -1,2 +1,20 @@ --- +# TODO(dawudm) We should replace these two tasks with a task to check the port +# is free in precheck.yml in the D release +- name: Stop and remove containers for haproxy exporter containers + become: true + kolla_container: + action: "stop_and_remove_container" + name: "prometheus_haproxy_exporter" + when: + - inventory_hostname in groups['loadbalancer'] + +- name: Removing config for haproxy exporter + file: + path: "{{ node_config_directory }}/prometheus-haproxy-exporter" + state: "absent" + become: true + when: + - inventory_hostname in groups['loadbalancer'] + - import_tasks: deploy.yml diff --git a/ansible/roles/loadbalancer/templates/haproxy/haproxy_main.cfg.j2 b/ansible/roles/loadbalancer/templates/haproxy/haproxy_main.cfg.j2 index 0e572333b9..ef4638d81e 100644 --- a/ansible/roles/loadbalancer/templates/haproxy/haproxy_main.cfg.j2 +++ b/ansible/roles/loadbalancer/templates/haproxy/haproxy_main.cfg.j2 @@ -42,6 +42,13 @@ listen stats stats realm Haproxy\ Stats stats auth {{ haproxy_user }}:{{ haproxy_password }} +{% if enable_prometheus_haproxy_exporter | bool %} +listen metrics + bind {{ api_interface_address }}:{{ prometheus_haproxy_exporter_port }} + mode http + http-request use-service prometheus-exporter if { path /metrics } +{% endif %} + frontend status bind {{ api_interface_address }}:{{ haproxy_monitor_port }} {% if api_interface_address != kolla_internal_vip_address %} diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml index 8818f19a2f..49519ae0f1 100644 --- a/ansible/roles/prometheus/defaults/main.yml +++ b/ansible/roles/prometheus/defaults/main.yml @@ -37,13 +37,6 @@ prometheus_services: image: "{{ prometheus_mysqld_exporter_image_full }}" volumes: "{{ prometheus_mysqld_exporter_default_volumes + prometheus_mysqld_exporter_extra_volumes }}" dimensions: "{{ prometheus_mysqld_exporter_dimensions }}" - prometheus-haproxy-exporter: - container_name: prometheus_haproxy_exporter - group: prometheus-haproxy-exporter - enabled: "{{ enable_prometheus_haproxy_exporter | bool }}" - image: "{{ prometheus_haproxy_exporter_image_full }}" - volumes: "{{ prometheus_haproxy_exporter_default_volumes + prometheus_haproxy_exporter_extra_volumes }}" - dimensions: "{{ prometheus_haproxy_exporter_dimensions }}" prometheus-memcached-exporter: container_name: prometheus_memcached_exporter group: prometheus-memcached-exporter @@ -197,10 +190,6 @@ prometheus_server_image: "{{ docker_registry ~ '/' if docker_registry else '' }} prometheus_server_tag: "{{ prometheus_tag }}" prometheus_server_image_full: "{{ prometheus_server_image }}:{{ prometheus_server_tag }}" -prometheus_haproxy_exporter_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/prometheus-haproxy-exporter" -prometheus_haproxy_exporter_tag: "{{ prometheus_tag }}" -prometheus_haproxy_exporter_image_full: "{{ prometheus_haproxy_exporter_image }}:{{ prometheus_haproxy_exporter_tag }}" - prometheus_mysqld_exporter_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/prometheus-mysqld-exporter" prometheus_mysqld_exporter_tag: "{{ prometheus_tag }}" prometheus_mysqld_exporter_image_full: "{{ prometheus_mysqld_exporter_image }}:{{ prometheus_mysqld_exporter_tag }}" @@ -243,7 +232,6 @@ prometheus_msteams_tag: "{{ prometheus_tag }}" prometheus_msteams_image_full: "{{ prometheus_msteams_image }}:{{ prometheus_msteams_tag }}" prometheus_server_dimensions: "{{ default_container_dimensions }}" -prometheus_haproxy_exporter_dimensions: "{{ default_container_dimensions }}" prometheus_mysqld_exporter_dimensions: "{{ default_container_dimensions }}" prometheus_node_exporter_dimensions: "{{ default_container_dimensions }}" prometheus_memcached_exporter_dimensions: "{{ default_container_dimensions }}" @@ -261,12 +249,6 @@ prometheus_server_default_volumes: - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}" - "prometheus_v2:/var/lib/prometheus" - "kolla_logs:/var/log/kolla/" -prometheus_haproxy_exporter_default_volumes: - - "{{ node_config_directory }}/prometheus-haproxy-exporter/:{{ container_config_directory }}/:ro" - - "/etc/localtime:/etc/localtime:ro" - - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}" - - "kolla_logs:/var/log/kolla/" - - "haproxy_socket:/var/lib/kolla/haproxy" prometheus_mysqld_exporter_default_volumes: - "{{ node_config_directory }}/prometheus-mysqld-exporter/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" @@ -327,7 +309,6 @@ prometheus_msteams_default_volumes: prometheus_extra_volumes: "{{ default_extra_volumes }}" prometheus_server_extra_volumes: "{{ prometheus_extra_volumes }}" -prometheus_haproxy_exporter_extra_volumes: "{{ prometheus_extra_volumes }}" prometheus_mysqld_exporter_extra_volumes: "{{ prometheus_extra_volumes }}" prometheus_node_exporter_extra_volumes: "{{ prometheus_extra_volumes }}" prometheus_memcached_exporter_extra_volumes: "{{ prometheus_extra_volumes }}" @@ -356,7 +337,6 @@ prometheus_server_command: >- prometheus_blackbox_exporter_cmdline_extras: "" prometheus_cadvisor_cmdline_extras: "--docker_only --store_container_labels=false --disable_metrics=percpu,referenced_memory,cpu_topology,resctrl,udp,advtcp,sched,hugetlb,memory_numa,tcp,process" prometheus_elasticsearch_exporter_cmdline_extras: "" -prometheus_haproxy_exporter_cmdline_extras: "" prometheus_memcached_exporter_cmdline_extras: "" prometheus_mysqld_exporter_cmdline_extras: "" prometheus_node_exporter_cmdline_extras: "" diff --git a/ansible/roles/prometheus/handlers/main.yml b/ansible/roles/prometheus/handlers/main.yml index e9bccb5b43..dcdb7578d0 100644 --- a/ansible/roles/prometheus/handlers/main.yml +++ b/ansible/roles/prometheus/handlers/main.yml @@ -45,21 +45,6 @@ when: - kolla_action != "config" -- name: Restart prometheus-haproxy-exporter container - vars: - service_name: "prometheus-haproxy-exporter" - service: "{{ prometheus_services[service_name] }}" - become: true - kolla_container: - action: "recreate_or_restart_container" - common_options: "{{ docker_common_options }}" - name: "{{ service.container_name }}" - image: "{{ service.image }}" - volumes: "{{ service.volumes }}" - dimensions: "{{ service.dimensions }}" - when: - - kolla_action != "config" - - name: Restart prometheus-memcached-exporter container vars: service_name: "prometheus-memcached-exporter" diff --git a/ansible/roles/prometheus/tasks/precheck.yml b/ansible/roles/prometheus/tasks/precheck.yml index 744bc4ad7c..29b71e3167 100644 --- a/ansible/roles/prometheus/tasks/precheck.yml +++ b/ansible/roles/prometheus/tasks/precheck.yml @@ -12,7 +12,6 @@ name: - prometheus_server - prometheus_node_exporter - - prometheus_haproxy_exporter - prometheus_memcached_exporter - prometheus_mysqld_exporter - prometheus_cadvisor @@ -83,18 +82,6 @@ - inventory_hostname in groups['prometheus-mysqld-exporter'] - enable_prometheus_mysqld_exporter | bool -- name: Checking free port for Prometheus haproxy_exporter - wait_for: - host: "{{ 'api' | kolla_address }}" - port: "{{ prometheus_haproxy_exporter_port }}" - connect_timeout: 1 - timeout: 1 - state: stopped - when: - - container_facts['prometheus_haproxy_exporter'] is not defined - - inventory_hostname in groups['prometheus-haproxy-exporter'] - - enable_prometheus_haproxy_exporter | bool - - name: Checking free port for Prometheus memcached_exporter wait_for: host: "{{ 'api' | kolla_address }}" diff --git a/ansible/roles/prometheus/templates/prometheus-haproxy-exporter.json.j2 b/ansible/roles/prometheus/templates/prometheus-haproxy-exporter.json.j2 deleted file mode 100644 index c6d43fa5b8..0000000000 --- a/ansible/roles/prometheus/templates/prometheus-haproxy-exporter.json.j2 +++ /dev/null @@ -1,11 +0,0 @@ -{ - "command": "/opt/haproxy_exporter/haproxy_exporter --haproxy.scrape-uri unix:/var/lib/kolla/haproxy/haproxy.sock --web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_haproxy_exporter_port }}{% if prometheus_haproxy_exporter_cmdline_extras %} {{ prometheus_haproxy_exporter_cmdline_extras }}{% endif %}", - "config_files": [], - "permissions": [ - { - "path": "/var/log/kolla/prometheus", - "owner": "prometheus:kolla", - "recurse": true - } - ] -} diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 index be4febed04..f69281b6ff 100644 --- a/ansible/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -60,9 +60,9 @@ scrape_configs: {% if enable_prometheus_haproxy_exporter | bool %} - job_name: haproxy static_configs: -{% for host in groups['prometheus-haproxy-exporter'] %} +{% for host in groups['loadbalancer'] %} - targets: - - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_haproxy_exporter_port'] }}' + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_haproxy_exporter_port }}' {% if hostvars[host].prometheus_instance_label | default(false, true) %} labels: instance: "{{ hostvars[host].prometheus_instance_label }}" diff --git a/ansible/site.yml b/ansible/site.yml index 34deb28b5a..2f72bdd2a7 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -452,7 +452,6 @@ - prometheus - prometheus-node-exporter - prometheus-mysqld-exporter - - prometheus-haproxy-exporter - prometheus-memcached-exporter - prometheus-cadvisor - prometheus-alertmanager diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 3429c2c0e4..f19fd6c636 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -792,7 +792,6 @@ workaround_ansible_issue_8743: yes # Extra parameters passed to Prometheus exporters. #prometheus_blackbox_exporter_cmdline_extras: #prometheus_elasticsearch_exporter_cmdline_extras: -#prometheus_haproxy_exporter_cmdline_extras: #prometheus_memcached_exporter_cmdline_extras: #prometheus_mysqld_exporter_cmdline_extras: #prometheus_node_exporter_cmdline_extras: diff --git a/releasenotes/notes/haproxy-metrics-endpoint-d62c0f34e29a10b2.yaml b/releasenotes/notes/haproxy-metrics-endpoint-d62c0f34e29a10b2.yaml new file mode 100644 index 0000000000..ef3db107b6 --- /dev/null +++ b/releasenotes/notes/haproxy-metrics-endpoint-d62c0f34e29a10b2.yaml @@ -0,0 +1,19 @@ +--- +features: + - | + Removed configuration and deployment of ``prometheus-haproxy-exporter`` as + its repository is now archived. We now use the native support for + Prometheus which is now built into HAProxy. For consistency this is exposed + on the ``prometheus_haproxy_exporter_port`` port. + ``prometheus-haproxy-exporter`` containers and config are automatically + removed. + +upgrade: + - | + Due to the change from using the ``prometheus-haproxy-exporter`` to using + the native support for Prometheus which is now built into HAProxy, metric + names may have been replaced and/or removed, and in some cases the metric + names may have remained the same but the labels may have changed. Alerts + and dashboards may also need to be updated to use the new metrics. Please + review any configuration that references the old metrics as this is not a + backwards compatible change. diff --git a/specs/prometheus.rst b/specs/prometheus.rst index ad0646b602..e4a20a6c95 100644 --- a/specs/prometheus.rst +++ b/specs/prometheus.rst @@ -261,9 +261,9 @@ As with all optional services in Kolla Ansible, Prometheus deployment should be controlled by Kolla Ansible variables. A high level enable_prometheus variable should control whether Prometheus is used at all. Additionally, additional variables can be used to control individual exporters. For example, -enable_prometheus_haproxy could be used to enable/disable the HAProxy exporter -to Prometheus. By default Prometheus should be enabled and exporters should be -enabled if both Prometheus and the associated service are enabled. +enable_prometheus_haproxy_exporter could be used to enable/disable the HAProxy +exporter to Prometheus. By default Prometheus should be enabled and exporters +should be enabled if both Prometheus and the associated service are enabled. Limitations ----------- diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2 index e1b1222884..74f5b21b34 100644 --- a/tests/templates/inventory.j2 +++ b/tests/templates/inventory.j2 @@ -681,9 +681,6 @@ storage [prometheus-mysqld-exporter:children] mariadb -[prometheus-haproxy-exporter:children] -loadbalancer - [prometheus-memcached-exporter:children] memcached