diff --git a/doc/source/logstash.rst b/doc/source/logstash.rst index 5fd7ea918a..630ec57fbd 100644 --- a/doc/source/logstash.rst +++ b/doc/source/logstash.rst @@ -75,16 +75,16 @@ bottleneck very quickly. This looks something like: zuul post-logs playbook | | - gearman-client --------------- - / | \ | - / | \ | - gearman gearman gearman subunit gearman - worker1 worker2 worker3 worker01 - | | | | - logstash logstash logstash | - indexer1 indexer2 indexer3 | - \ | / subunit2sql - \ | / DB + gearman-client + / | \ + / | \ + gearman gearman gearman + worker1 worker2 worker3 + | | | + logstash logstash logstash + indexer1 indexer2 indexer3 + \ | / + \ | / elasticsearch cluster | @@ -113,19 +113,6 @@ can be found at * https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/log-gearman-client.py * https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/log-gearman-worker.py -Subunit Gearman Worker ----------------------- - -Using the same mechanism as the Log pushers there is an additional class of -gearman worker that takes the subunit output from test runs and stores them in -a subunit2SQL database. Right now this is only done with the subunit output -from gate and periodic queue tempest runs. - -If you are interested in technical details the source of this script can be -found at: - -* https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/subunit-gearman-worker.py - Logstash -------- @@ -215,27 +202,6 @@ be reached at http://logstash.openstack.org. See :ref:`query-logstash` for more info on using Kibana to perform queries. -subunit2SQL ------------ -subunit2SQL is a python project for taking subunit v2 streams and storing them -in a SQL database. More information on the subunit protocol can be found here: -https://github.com/testing-cabal/subunit/blob/master/README - -subunit2sql provides a database schema, several utilities for interacting with -the database, and a python library to build tooling on top of the database. -More information about using subunit2sql can be found at: -http://docs.openstack.org/developer/subunit2sql/ - -Our instance of the subunit2SQL database is running on a MySQL database and has -been configured to be remotely accessible to allow for public querying. The -public query access is provided with the following credentials:: - - username: query - password: query - hostname: logstash.openstack.org - database name: subunit2sql - database port: 3306 - simpleproxy ----------- Simpleproxy is a simple tcp proxy which allows forwarding tcp connections from diff --git a/hiera/common.yaml b/hiera/common.yaml index 5e43077428..5eadefe33a 100644 --- a/hiera/common.yaml +++ b/hiera/common.yaml @@ -33,7 +33,6 @@ cacti_hosts: - gitea08.opendev.org - grafana01.opendev.org - graphite02.opendev.org -- health.openstack.org - jvb01.opendev.org - jvb02.opendev.org - kdc03.openstack.org @@ -94,8 +93,6 @@ cacti_hosts: - status01.openstack.org - storyboard01.opendev.org - storyboard-dev01.opendev.org -- subunit-worker01.openstack.org -- subunit-worker02.openstack.org - translate.openstack.org - translate-dev.openstack.org - wiki.openstack.org diff --git a/inventory/service/group_vars/logstash.yaml b/inventory/service/group_vars/logstash.yaml index 91e4cf90d0..4295a978c1 100644 --- a/inventory/service/group_vars/logstash.yaml +++ b/inventory/service/group_vars/logstash.yaml @@ -3,5 +3,4 @@ iptables_extra_public_tcp_ports: - 3306 iptables_extra_allowed_groups: - {'protocol': 'tcp', 'port': '4730', 'group': 'logstash-worker'} - - {'protocol': 'tcp', 'port': '4730', 'group': 'subunit-worker'} - {'protocol': 'tcp', 'port': '4730', 'group': 'zuul-executor'} diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml index 66754358d1..e2037f961f 100644 --- a/inventory/service/groups.yaml +++ b/inventory/service/groups.yaml @@ -139,7 +139,6 @@ groups: - status*.open*.org - storyboard-dev[0-9]*.opendev.org - storyboard[0-9]*.opendev.org - - subunit-worker[0-9]*.open*.org - translate-dev[0-9]*.open*.org - translate[0-9]*.open*.org puppet4: @@ -152,7 +151,6 @@ groups: - status*.open*.org - storyboard[0-9]*.opendev.org - storyboard-dev[0-9]*.opendev.org - - subunit-worker[0-9]*.open*.org - translate[0-9]*.open*.org - translate-dev[0-9]*.open*.org refstack: @@ -172,8 +170,6 @@ groups: - storyboard[0-9]*.opendev.org storyboard-dev: - storyboard-dev[0-9]*.opendev.org - subunit-worker: - - subunit-worker[0-9]*.open*.org translate-dev: - translate-dev[0-9]*.open*.org translate: diff --git a/manifests/site.pp b/manifests/site.pp index c3220b1e35..41fe4d31d0 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -61,18 +61,6 @@ node /^logstash-worker\d+\.open.*\.org$/ { } } -# Node-OS: xenial -node /^subunit-worker\d+\.open.*\.org$/ { - $group = "subunit-worker" - class { 'openstack_project::server': } - class { 'openstack_project::subunit_worker': - subunit2sql_db_host => hiera('subunit2sql_db_host', ''), - subunit2sql_db_pass => hiera('subunit2sql_db_password', ''), - mqtt_pass => hiera('mqtt_service_user_password'), - mqtt_ca_cert_contents => hiera('mosquitto_tls_ca_file'), - } -} - # Node-OS: xenial node /^elasticsearch\d+\.open.*\.org$/ { $group = "elasticsearch" diff --git a/modules/openstack_project/manifests/subunit_worker.pp b/modules/openstack_project/manifests/subunit_worker.pp deleted file mode 100644 index f789e1283e..0000000000 --- a/modules/openstack_project/manifests/subunit_worker.pp +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# subunit2sql worker glue class. -# -class openstack_project::subunit_worker ( - $subunit2sql_db_host, - $subunit2sql_db_pass, - $mqtt_user = 'infra', - $mqtt_pass = undef, - $mqtt_ca_cert_contents = undef, -) { - - file { '/etc/subunit2sql/subunit-woker.yaml': - ensure => file, - owner => 'root', - group => 'root', - mode => '0555', - content => template('openstack_project/logstash/jenkins-subunit-worker.yaml.erb'), - } - - file { '/etc/subunit2sql/mqtt-root-CA.pem.crt': - ensure => present, - content => $mqtt_ca_cert_contents, - replace => true, - owner => 'subunit', - group => 'subunit', - mode => '0555', - } - - include subunit2sql - subunit2sql::worker { 'A': - config_file => '/etc/subunit2sql/subunit-woker.yaml', - db_host => $subunit2sql_db_host, - db_pass => $subunit2sql_db_pass, - } - -} diff --git a/modules/openstack_project/templates/logstash/jenkins-subunit-worker.yaml.erb b/modules/openstack_project/templates/logstash/jenkins-subunit-worker.yaml.erb deleted file mode 100644 index 5d5bb7a954..0000000000 --- a/modules/openstack_project/templates/logstash/jenkins-subunit-worker.yaml.erb +++ /dev/null @@ -1,9 +0,0 @@ -gearman-host: logstash.openstack.org -gearman-port: 4730 -config: /etc/subunit2sql/subunit2sql.conf -#mqtt-host: firehose.openstack.org -#mqtt-port: 8883 -#mqtt-topic: gearman-subunit/<%= @hostname %> -#mqtt-user: <%= @mqtt_user %> -#mqtt-pass: <%= @mqtt_pass %> -#mqtt-ca-certs: /etc/subunit2sql/mqtt-root-CA.pem.crt