Merge "Remove config management for subunit2sql workers"
This commit is contained in:
Zuul
Gerrit Code Review
commit
4e8d344e67
|
|
@ -75,16 +75,16 @@ bottleneck very quickly. This looks something like:
|
|||
zuul post-logs playbook
|
||||
|
|
||||
|
|
||||
gearman-client ---------------
|
||||
/ | \ |
|
||||
/ | \ |
|
||||
gearman gearman gearman subunit gearman
|
||||
worker1 worker2 worker3 worker01
|
||||
| | | |
|
||||
logstash logstash logstash |
|
||||
indexer1 indexer2 indexer3 |
|
||||
\ | / subunit2sql
|
||||
\ | / DB
|
||||
gearman-client
|
||||
/ | \
|
||||
/ | \
|
||||
gearman gearman gearman
|
||||
worker1 worker2 worker3
|
||||
| | |
|
||||
logstash logstash logstash
|
||||
indexer1 indexer2 indexer3
|
||||
\ | /
|
||||
\ | /
|
||||
elasticsearch
|
||||
cluster
|
||||
|
|
||||
|
|
@ -113,19 +113,6 @@ can be found at
|
|||
* https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/log-gearman-client.py
|
||||
* https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/log-gearman-worker.py
|
||||
|
||||
Subunit Gearman Worker
|
||||
----------------------
|
||||
|
||||
Using the same mechanism as the Log pushers there is an additional class of
|
||||
gearman worker that takes the subunit output from test runs and stores them in
|
||||
a subunit2SQL database. Right now this is only done with the subunit output
|
||||
from gate and periodic queue tempest runs.
|
||||
|
||||
If you are interested in technical details the source of this script can be
|
||||
found at:
|
||||
|
||||
* https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/subunit-gearman-worker.py
|
||||
|
||||
Logstash
|
||||
--------
|
||||
|
||||
|
|
@ -215,27 +202,6 @@ be reached at http://logstash.openstack.org. See
|
|||
:ref:`query-logstash` for more info on using Kibana to perform
|
||||
queries.
|
||||
|
||||
subunit2SQL
|
||||
-----------
|
||||
subunit2SQL is a python project for taking subunit v2 streams and storing them
|
||||
in a SQL database. More information on the subunit protocol can be found here:
|
||||
https://github.com/testing-cabal/subunit/blob/master/README
|
||||
|
||||
subunit2sql provides a database schema, several utilities for interacting with
|
||||
the database, and a python library to build tooling on top of the database.
|
||||
More information about using subunit2sql can be found at:
|
||||
http://docs.openstack.org/developer/subunit2sql/
|
||||
|
||||
Our instance of the subunit2SQL database is running on a MySQL database and has
|
||||
been configured to be remotely accessible to allow for public querying. The
|
||||
public query access is provided with the following credentials::
|
||||
|
||||
username: query
|
||||
password: query
|
||||
hostname: logstash.openstack.org
|
||||
database name: subunit2sql
|
||||
database port: 3306
|
||||
|
||||
simpleproxy
|
||||
-----------
|
||||
Simpleproxy is a simple tcp proxy which allows forwarding tcp connections from
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ cacti_hosts:
|
|||
- gitea08.opendev.org
|
||||
- grafana01.opendev.org
|
||||
- graphite02.opendev.org
|
||||
- health.openstack.org
|
||||
- jvb01.opendev.org
|
||||
- jvb02.opendev.org
|
||||
- kdc03.openstack.org
|
||||
|
|
@ -94,8 +93,6 @@ cacti_hosts:
|
|||
- status01.openstack.org
|
||||
- storyboard01.opendev.org
|
||||
- storyboard-dev01.opendev.org
|
||||
- subunit-worker01.openstack.org
|
||||
- subunit-worker02.openstack.org
|
||||
- translate.openstack.org
|
||||
- translate-dev.openstack.org
|
||||
- wiki.openstack.org
|
||||
|
|
|
|||
|
|
@ -3,5 +3,4 @@ iptables_extra_public_tcp_ports:
|
|||
- 3306
|
||||
iptables_extra_allowed_groups:
|
||||
- {'protocol': 'tcp', 'port': '4730', 'group': 'logstash-worker'}
|
||||
- {'protocol': 'tcp', 'port': '4730', 'group': 'subunit-worker'}
|
||||
- {'protocol': 'tcp', 'port': '4730', 'group': 'zuul-executor'}
|
||||
|
|
|
|||
|
|
@ -139,7 +139,6 @@ groups:
|
|||
- status*.open*.org
|
||||
- storyboard-dev[0-9]*.opendev.org
|
||||
- storyboard[0-9]*.opendev.org
|
||||
- subunit-worker[0-9]*.open*.org
|
||||
- translate-dev[0-9]*.open*.org
|
||||
- translate[0-9]*.open*.org
|
||||
puppet4:
|
||||
|
|
@ -152,7 +151,6 @@ groups:
|
|||
- status*.open*.org
|
||||
- storyboard[0-9]*.opendev.org
|
||||
- storyboard-dev[0-9]*.opendev.org
|
||||
- subunit-worker[0-9]*.open*.org
|
||||
- translate[0-9]*.open*.org
|
||||
- translate-dev[0-9]*.open*.org
|
||||
refstack:
|
||||
|
|
@ -172,8 +170,6 @@ groups:
|
|||
- storyboard[0-9]*.opendev.org
|
||||
storyboard-dev:
|
||||
- storyboard-dev[0-9]*.opendev.org
|
||||
subunit-worker:
|
||||
- subunit-worker[0-9]*.open*.org
|
||||
translate-dev:
|
||||
- translate-dev[0-9]*.open*.org
|
||||
translate:
|
||||
|
|
|
|||
|
|
@ -61,18 +61,6 @@ node /^logstash-worker\d+\.open.*\.org$/ {
|
|||
}
|
||||
}
|
||||
|
||||
# Node-OS: xenial
|
||||
node /^subunit-worker\d+\.open.*\.org$/ {
|
||||
$group = "subunit-worker"
|
||||
class { 'openstack_project::server': }
|
||||
class { 'openstack_project::subunit_worker':
|
||||
subunit2sql_db_host => hiera('subunit2sql_db_host', ''),
|
||||
subunit2sql_db_pass => hiera('subunit2sql_db_password', ''),
|
||||
mqtt_pass => hiera('mqtt_service_user_password'),
|
||||
mqtt_ca_cert_contents => hiera('mosquitto_tls_ca_file'),
|
||||
}
|
||||
}
|
||||
|
||||
# Node-OS: xenial
|
||||
node /^elasticsearch\d+\.open.*\.org$/ {
|
||||
$group = "elasticsearch"
|
||||
|
|
|
|||
|
|
@ -1,49 +0,0 @@
|
|||
# Copyright 2013 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
# subunit2sql worker glue class.
|
||||
#
|
||||
class openstack_project::subunit_worker (
|
||||
$subunit2sql_db_host,
|
||||
$subunit2sql_db_pass,
|
||||
$mqtt_user = 'infra',
|
||||
$mqtt_pass = undef,
|
||||
$mqtt_ca_cert_contents = undef,
|
||||
) {
|
||||
|
||||
file { '/etc/subunit2sql/subunit-woker.yaml':
|
||||
ensure => file,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0555',
|
||||
content => template('openstack_project/logstash/jenkins-subunit-worker.yaml.erb'),
|
||||
}
|
||||
|
||||
file { '/etc/subunit2sql/mqtt-root-CA.pem.crt':
|
||||
ensure => present,
|
||||
content => $mqtt_ca_cert_contents,
|
||||
replace => true,
|
||||
owner => 'subunit',
|
||||
group => 'subunit',
|
||||
mode => '0555',
|
||||
}
|
||||
|
||||
include subunit2sql
|
||||
subunit2sql::worker { 'A':
|
||||
config_file => '/etc/subunit2sql/subunit-woker.yaml',
|
||||
db_host => $subunit2sql_db_host,
|
||||
db_pass => $subunit2sql_db_pass,
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
gearman-host: logstash.openstack.org
|
||||
gearman-port: 4730
|
||||
config: /etc/subunit2sql/subunit2sql.conf
|
||||
#mqtt-host: firehose.openstack.org
|
||||
#mqtt-port: 8883
|
||||
#mqtt-topic: gearman-subunit/<%= @hostname %>
|
||||
#mqtt-user: <%= @mqtt_user %>
|
||||
#mqtt-pass: <%= @mqtt_pass %>
|
||||
#mqtt-ca-certs: /etc/subunit2sql/mqtt-root-CA.pem.crt
|
||||
Loading…
Reference in New Issue