opendev
/
elastic-recheck
Code Issues Proposed changes

Merge remote-tracking branch 'origin/rdo' into merge-branch 

Change-Id: Iacac232d89950de29cc7b6aceea99b6ac0e8a176
This commit is contained in:
Dariusz Smigiel 2022-06-23 12:42:48 -07:00
parent 01b6b8299b 851b9e0a48
commit 2718aaa965
46 changed files with 720 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.gitignore vendored
View File

@ -53,3 +53,12 @@ ChangeLog
# Local data to test web-served html pages
web/share/elastic-recheck
web/share/data
# logs
data/*.log
data/id_ecdsa
data/id_ecdsa.pub
# local config
local/

1
.zuul.yaml
View File

@ -28,6 +28,7 @@
- job:
name: elastic-recheck-container
voting: false
parent: opendev-build-docker-image
description: Build container images for elastic-recheck service
vars:

65
Dockerfile
View File

@ -1,18 +1,65 @@
# syntax=docker/dockerfile:experimental
FROM opendevorg/python-builder:3.7 as elastic-recheck-builder
# We make use of alpine as it seems to have a more container friendly cron
# later alpine versions require different nginx ocnfiguration
FROM alpine:3.13 as elastic-recheck
# FROM opendevorg/python-builder:3.7 as elastic-recheck-builder
RUN \
apk update && \
apk add \
g++ \
gcc \
git \
musl-dev \
nginx \
python3-dev \
py3-argparse \
py3-babel \
py3-certifi \
py3-cffi \
py3-cryptography \
py3-distro \
py3-elasticsearch \
py3-httplib2 \
py3-jinja2 \
py3-netaddr \
py3-netifaces \
py3-oauthlib \
py3-paramiko \
py3-pbr \
py3-pip \
py3-requests \
py3-simplejson \
py3-sqlalchemy \
py3-tempita \
py3-tz \
py3-yaml \
&& ln -f -s /data/cron/crontab /etc/crontabs/root
WORKDIR /tmp/src
COPY . /tmp/src
RUN assemble
RUN pip3 install .
FROM opendevorg/python-base:3.7 as elastic-recheck
# RUN assemble
# FROM opendevorg/python-base:3.7 as elastic-recheck
# COPY --from=elastic-recheck-builder /output/ /output
COPY --from=elastic-recheck-builder /output/ /output
RUN /output/install-from-bindep && \
rm -rf /output
COPY data/ /data/
COPY queries/ /opt/elastic-recheck/queries
# RUN /output/install-from-bindep && \
# rm -rf /output && \
RUN rm -rf /tmp/src && \
mkdir -p /root/.ssh /data /run/nginx && \
chmod 700 /root/.ssh
COPY web/conf/nginx.conf /etc/nginx/conf.d/default.conf
COPY web/share/ /var/www/localhost
COPY data/cron/ /root/
COPY data/elastic-recheck.conf /root/elastic-recheck.conf
COPY data/recheckwatchbot.yaml /root/recheckwatchbot.yaml
COPY tools/ssh-check.py /root/ssh-check.py
# COPY data/crontab /var/spool/cron/crontabs/root
COPY data/id_ecdsa /root/.ssh/id_ecdsa
# using root allows us to use same relative paths in configs for running outside
# containers, where ./data contains persistent configs and logs.
WORKDIR /
CMD /usr/local/bin/elastic-recheck -f data/elastic-recheck.conf ${ER_OPTS:-}
CMD /usr/bin/elastic-recheck -f /root/elastic-recheck.conf ${ER_OPTS:-}

63
Makefile
View File

@ -4,11 +4,16 @@ PYTHON ?= $(shell command -v python3 python|head -n1)
# Keep docker before podman due to:
# https://github.com/containers/podman/issues/7602
ENGINE ?= $(shell command -v docker podman|head -n1)
COMPOSE ?= $(shell command -v docker-compose podman-compose|head -n1)
# localhost/ prefix must be present in order to assure docker/podman compatibility:
# https://github.com/containers/buildah/issues/1034
IMAGE_TAG=localhost/elastic-recheck
# Enable newer docker buildkit if available
DOCKER_BUILDKIT=1
COMPOSE_DOCKER_CLI_BUILD=1
# ssh opts to add, used only for testing
SSH_OPTS=-o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null"
.PHONY: default
default: help
@ -32,16 +37,66 @@ export PRINT_HELP_PYSCRIPT
help:
@$(PYTHON) -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
.PHONY: check-env
check-env:
ifndef GERRIT_USER
$(error GERRIT_USER is undefined, you need to define it to run this command)
endif
.PHONY: build
build: ## Build image using docker
build: data/id_ecdsa check-env ## Build image using $(ENGINE)
@echo "Checking that current user can connect to gerit using ssh..."""
@python3 ./tools/ssh-check.py
$(ENGINE) build -t $(IMAGE_TAG) .
@echo "Image size: $$(docker image inspect --format='scale=0; {{.Size}}/1024/1024' $(IMAGE_TAG) | bc)MB"
@echo "Image size: $$($(ENGINE) image inspect --format='scale=0; {{.Size}}/1024/1024' $(IMAGE_TAG) | bc)MB"
@echo "Validate that built container can also connect to gerrit..."""
$(ENGINE) run --env GERRIT_USER -it $(IMAGE_TAG) python3 /root/ssh-check.py
.PHONY: up
up: data/id_ecdsa check-env ## Run containers
@# validates that container has credentials and connectivity to talk with gerrit server
@# Validate the builder image can connect to server
@# $(ENGINE) run --env GERRIT_USER -it $(IMAGE_TAG) python3 /root/ssh-check.py
@# --abort-on-container-exit
$(COMPOSE) up --force-recreate --remove-orphans --abort-on-container-exit <<<y
@# $(ENGINE) container stop elastic-recheck
@# bash -c "$(ENGINE) rm elastic-recheck || true"
@# $(ENGINE) run --env ES_URL --env LS_URL --env DB_URI --env GERRIT_HOST --env GERRIT_USER --env IRC_NICK --env IRC_PASS -it --name elastic-recheck elastic-recheck:latest
.PHONY: down
down: ## Destroy containers
$(COMPOSE) down --remove-orphans --rmi local
.PHONY: into
into: ## Starts bash inside docker image
$(ENGINE) run -it $(IMAGE_TAG) /bin/bash
into: ## Starts bash inside $(ENGINE) image
$(ENGINE) run --mount 'type=volume,src=er-volume,dst=/data,volume-driver=local' -it $(IMAGE_TAG) /bin/sh
# $(ENGINE) exec -it er-cron /bin/sh
@# $(COMPOSE) run elastic-recheck bash
.PHONY: dive
dive: ## Use `dive` tool to investigate container size
# https://github.com/wagoodman/dive
dive $(IMAGE_TAG)
data/id_ecdsa:
# this key must be unencrypted, so create a spare one for testing and
# add it to your gerrit user configuration
cp -f ~/.ssh/id_ecdsa_insecure data/id_ecdsa
.PHONY: clean
clean: ## Use clean to remove all temp files, including container and images but **not** data/
$(ENGINE) image rm --force $(IMAGE_TAG)
.PHONY: key
key: ## Makes a SSH key compatibile with paramiko (overrides existing one)
@mkdir -p data
@rm data/id_ecdsa || true
@ssh-keygen -q -N '' -m PEM -t ecdsa -f data/id_ecdsa
@echo "WARN: Please assign key below to your gerrit user using the web interface:"
@ssh-keygen -l -f data/id_ecdsa
.PHONY: wheel
wheel: ## Use clean to remove all temp files, including container and images but **not** data/
mkdir -p build
cd build && pip3 wheel ..

17
README.rst
View File

@ -229,6 +229,23 @@ You can execute an individual query locally and analyze the search results::
95% master
4% stable/icehouse
Container
---------
Elastic Recheck container is a newer deployment model that use a docker-compose
file that deploys the 3 services using the same container. They all share a
single persistent ``er-volume``.
The services are:
* ``er-bot`` - This is watching gerrit even stream for new events
* ``er-cron`` - This refreshes the 3 json files on a schedule
* ``er-web`` - Provides the dashboard frontend
A developer should easily start all of these by running ``make up``. By default
if any of the services is failing all containers will be stopped.
Notes
-----

1
bindep.txt
View File

@ -16,3 +16,4 @@ libopenssl-devel [platform:suse !platform:rpm test compile]
locales [platform:debian]
python3-dev [platform:dpkg test compile]
python3-devel [platform:rpm test compile]
python3-cryptography [platform:rpm]

5
data/cron/cron-er-graph-all.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/sh
set -ex
cd /data/www
/root/er-safe-run.sh elastic-recheck-graph /opt/tripleo-ci-health-queries/output/elastic-recheck -o all-new.json
mv all-new.json all.json

4
data/cron/cron-er-graph-gate.sh Executable file
View File

@ -0,0 +1,4 @@
#!/bin/sh
set -ex
cd /data/www
/root/er-safe-run.sh elastic-recheck-graph /opt/tripleo-ci-health-queries/output/elastic-recheck -o gate-new.json -q gate && mv gate-new.json gate.json

6
data/cron/cron-er-graph-uncat.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/sh
set -ex
cd /data/www
mkdir -p new
/root/er-safe-run.sh elastic-recheck-uncategorized -d /opt/tripleo-ci-health-queries/output/elastic-recheck -t /var/www/localhost/templates -o new
mv new/*.html .

17
data/cron/cron-er-queries.sh Executable file
View File

@ -0,0 +1,17 @@
#! /bin/sh
set -ex
# Check if elastic-recheck is indeed a git repository
if [ ! -d /opt/tripleo-ci-health-queries ]; then
cd /opt
git clone https://opendev.org/openstack/tripleo-ci-health-queries
else
cd /opt/tripleo-ci-health-queries
git pull
if [! $? -eq 0]; then
cd /opt
rm -rf /opt/tripleo-ci-health-queries
git clone https://opendev.org/openstack/tripleo-ci-health-queries
fi
fi

14
data/cron/cron-start.sh Executable file
View File

@ -0,0 +1,14 @@
#!/bin/sh
set -ex
# er-volume may be empty at start
mkdir -p /data/www
# Used under docker to emulate cron execution, but we force running all
# jobs first, so we do not have to wait for refresh when newly deployed.
/root/cron-er-queries.sh
/root/cron-er-graph-uncat.sh
/root/cron-er-graph-gate.sh
/root/cron-er-graph-all.sh
# Run cron in foreground foreever:
crond -f -d 2

5
data/cron/crontab Normal file
View File

@ -0,0 +1,5 @@
PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin
20,50 * * * * /root//root/cron-er-queries.sh
00,30 * * * * /root/cron-er-graph-all.sh
10,40 * * * * /root/cron-er-graph-gate.sh
20,50 * * * * /root/cron-er-graph-uncat.sh

5
data/cron/er-safe-run.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/sh
# Timeout after 4 hours as we've seen this script occasionally fail to
# exit after running for days.
flock /var/run/er-safe-run.lock timeout -s KILL 14400 $@

33
data/elastic-recheck.conf Normal file
View File

@ -0,0 +1,33 @@
# Relative paths use container WORKDIR (/) or repo root when outside container
# [DEFAULT]
# GERRIT_QUERY_FILE=/opt/elastic-recheck/queries
# IRC_NICK=elastic-recheck
# IRC_PASS=
# IRC_CHANNEL_CONFIG=recheckwatchbot.yaml
# LOG_CONFIG=data/logging.config
# JOBS_RE=(dsvm|tripleo|devstack|tempest|grenade)
[ircbot]
# nick=${IRC_NICK}
# pass=${IRC_PASS}
server=irc.oftc.net
port=6667
channel_config=recheckwatchbot.yaml
# log_config=${LOG_CONFIG}
[gerrit]
host=${GERRIT_HOST}
user=${GERRIT_USER}
query_file=${GERRIT_QUERY_FILE}
key=/root/.ssh/id_ecdsa
[data_source]
es_url=${ES_URL}
ls_url=${LS_URL}
db_uri=${DB_URI}
[recheckwatch]
# TODO(mriedem): With zuul v3 there is a wild west of job names so this
# regex is table stakes, but there are clearly things missing from this like
# the various nova-* jobs (nova-multiattach, nova-live-migration, nova-next).
jobs_re=(dsvm|tripleo|devstack|tempest|grenade)

1
data/id_ecdsa Normal file
View File

@ -0,0 +1 @@
This is a dummy file

52
data/logging.config Normal file
View File

@ -0,0 +1,52 @@
[loggers]
keys=root,recheckwatchbot,elastic-recheck,irc
[handlers]
keys=console,debug,normal
[formatters]
keys=simple
[logger_root]
level=DEBUG
handlers=console,debug,normal
[logger_elastic-recheck]
level=DEBUG
handlers=console,debug,normal
qualname=elastic-recheck
propagate=0
[logger_recheckwatchbot]
level=DEBUG
handlers=console,debug,normal
qualname=recheckwatchbot
propagate=0
[logger_irc]
level=DEBUG
handlers=console,debug,normal
qualname=irc
propagate=0
[handler_console]
level=WARNING
class=StreamHandler
formatter=simple
args=(sys.stdout,)
[handler_debug]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('data/elastic-recheck_debug.log', 'midnight', 1, 30,)
[handler_normal]
level=INFO
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('data/elastic-recheck.log', 'midnight', 1, 30,)
[formatter_simple]
format=%(asctime)s %(levelname)s %(name)s: %(message)s
datefmt=

38
data/recheckwatchbot.yaml Normal file
View File

@ -0,0 +1,38 @@
channels:
oooq:
projects:
# elastic-recheck doesn't allow to limit reports to patches from a specific repo,
# so let's at least scope to bugs that neutron team acknowledged ownership for
# in Launchpad
- neutron
events:
# to limit reports to failures that belong to bugs owned by neutron team;
# we may revisit it once elastic-recheck learns how to limit the scope of reports
# to particular repositories
- positive
oooq:
projects:
- all
events:
- positive
- negative
messages:
# | means don't fold newlines, > means do
found_bug: |
I noticed Zuul failed, I think you hit bug(s):
%(bugs)s
footer: >-
For more details on this and other bugs, please see
http://ci-health-rdo.tripleo.org/
recheck_instructions: >-
If you believe we've correctly identified the failure, feel free to leave a 'recheck'
comment to run the tests again.
unrecognized: >-
Some of the tests failed in a way that we did not understand. Please help
us classify these issues so that they can be part of Elastic Recheck
http://ci-health-rdo.tripleo.org/
no_bugs_found: >-
I noticed Zuul failed, refer to:
https://docs.openstack.org/infra/manual/developers.html#automated-testing

83
docker-compose.yml Normal file
View File

@ -0,0 +1,83 @@
version: '2'
services:
cron:
container_name: er-cron
image: localhost/elastic-recheck
# the first time it starts we want to update as fast as possible, and
# switch to cron afterwords.
command: /root/cron-start.sh
environment:
- DB_URI
- ES_URL
- GERRIT_HOST
# - GERRIT_KEY
- GERRIT_USER=${GERRIT_USER}
- IRC_NICK
- IRC_PASS
- LOG_CONFIG
- LS_URL
volumes:
- er-volume:/data
bot:
container_name: er-bot
image: localhost/elastic-recheck
working_dir: /root
command: /usr/bin/elastic-recheck --noirc -f elastic-recheck.conf
environment:
- DB_URI
- ES_URL
- GERRIT_HOST
# - GERRIT_KEY
- GERRIT_USER
- IRC_NICK
- IRC_PASS
- LOG_CONFIG
- LS_URL
volumes:
- er-volume:/data
web:
container_name: er-web
image: localhost/elastic-recheck
command: nginx -g 'daemon off;'
environment:
- DB_URI
- ES_URL
- GERRIT_HOST
# - GERRIT_KEY
- GERRIT_USER=${GERRIT_USER}
- IRC_NICK
- IRC_PASS
- LOG_CONFIG
- LS_URL
ports:
- 80:80
# we do not want to start it too soon as it may fail to start if
# the er-volume is empty
depends_on:
- cron
- bot
volumes:
- er-volume:/data
labels:
- traefik.enable=true
- traefik.port=80
- traefik.http.routers.er.rule=Host(`er.sbarnea.com`)
- traefik.http.routers.er.tls.certResolver=myresolver
- traefik.http.routers.er.entrypoints=websecure
- traefik.http.services.er.loadbalancer.server.port=80
# do not mention network_mode or we risk getting:
# conflicting options: host type networking can't be used with links
# network_mode: host
# networks:
# hostnet: {}
volumes:
er-volume:
name: er-volume
driver: local
# # networks:
# # hostnet:
# # external: true
# # name: host

8
elasticRecheck.conf.sample
View File

@ -4,7 +4,7 @@ nick=RecheckWatchBot
pass=
server=irc.freenode.net
port=6667
channel_config=/home/mtreinish/elasticRecheck/recheckwatchbot.yaml
channel_config=data/recheckwatchbot.yaml
[recheckwatch]
#Any project that has a job that matches this regex will have all their
@ -13,10 +13,10 @@ jobs_re=dsvm
ci_username=jenkins
[gerrit]
user=treinish
user=os-tripleo-ci
host=review.opendev.org
query_file=/home/mtreinish/elasticRecheck/queries
key=/home/mtreinish/.ssh/id_rsa
query_file=/opt/elastic-recheck/queries
key=/root/.ssh/id_ecdsa
[data_source]
es_url=http://logstash.openstack.org:80/elasticsearch

13
elastic_recheck/bot.py
View File

@ -209,16 +209,16 @@ class RecheckWatch(threading.Thread):
if not event.get_all_bugs():
self._read(event)
else:
self._read(event)
stream.leave_comment(
event,
self.msgs,
debug=not self.commenting)
except er.ResultTimedOut as e:
self.log.warning(e.message)
self._read(msg=e.message)
except Exception:
self.log.exception("Uncaught exception processing event.")
self.log.warning(e.args[0])
self._read(msg=e.args[0])
except Exception as exp:
self.log.exception("Uncaught exception processing event: %s",
str(exp))
class MessageConfig(dict):
@ -237,7 +237,7 @@ class ChannelConfig(object):
self.data = data
keys = data.keys()
for key in keys:
for key in list(keys):
if key[0] != '#':
data['#' + key] = data.pop(key)
self.channels = data.keys()
@ -291,6 +291,7 @@ def _main(args, config):
msgs = MessageConfig(yaml.safe_load(open(fp)))
if not args.noirc:
print(dir(config))
bot = RecheckWatchBot(
channel_config.channels,
config=config)

0
elastic_recheck/cmd/__init__.py
View File

5
elastic_recheck/cmd/check_success.py
View File

@ -52,7 +52,8 @@ def all_fails(classifier):
so we can figure out how good we are doing on total classification.
"""
all_fails = {}
results = classifier.hits_by_query(er_config.ALL_FAILS_QUERY, size=30000)
results = classifier.hits_by_query(er_config.ALL_FAILS_QUERY,
size=10000, days=7)
facets = er_results.FacetSet()
facets.detect_facets(results, ["build_uuid"])
for build in facets:
@ -165,7 +166,7 @@ def collect_metrics(classifier, fails):
data = {}
for q in classifier.queries:
start = time.time()
results = classifier.hits_by_query(q['query'], size=30000)
results = classifier.hits_by_query(q['query'], size=10000, days=7)
log = logging.getLogger('recheckwatchbot')
log.debug("Took %d seconds to run (uncached) query for bug %s",
time.time() - start, q['bug'])

20
elastic_recheck/cmd/graph.py
View File

@ -18,9 +18,9 @@ import argparse
from datetime import datetime
import json
import os
from lazr.restfulclient.errors import ServerError
from launchpadlib import launchpad
import pyelasticsearch
import elasticsearch
import pytz
import requests
@ -71,6 +71,11 @@ def get_launchpad_bug(bug):
LOG.exception("Failed to get Launchpad data for bug %s", bug)
bugdata = dict(name='Unable to get launchpad data',
affects='Unknown', reviews=[])
# because for some reason launchpad returns 500 instead of 404
except ServerError:
LOG.exception("Failed to get Launchpad data for bug %s", bug)
bugdata = dict(name='Unable to get launchpad data',
affects='Unknown', reviews=[])
return bugdata
@ -149,7 +154,8 @@ def main():
timeframe = days * 24 * STEP / 1000
last_indexed = int(
((classifier.most_recent() - epoch).total_seconds()) * 1000)
((classifier.most_recent().replace(tzinfo=pytz.utc)
- epoch).total_seconds()) * 1000)
behind = now - last_indexed
# the data we're going to return, including interesting headers
@ -161,9 +167,6 @@ def main():
}
# Get the cluster health for the header
es = pyelasticsearch.ElasticSearch(config.es_url)
jsondata['status'] = es.health()['status']
for query in classifier.queries:
if args.queue:
query['query'] += ' AND build_queue:%s' % args.queue
@ -186,6 +189,7 @@ def main():
fails=0,
fails24=0,
data=[],
msg=query.get('msg') or query['query'],
voting=(not query.get('allow-nonvoting')))
buglist.append(bug)
try:
@ -193,7 +197,7 @@ def main():
args.queue,
size=3000,
days=days)
except pyelasticsearch.exceptions.InvalidJsonResponseError:
except elasticsearch.SerializationError:
LOG.exception("Invalid Json while collecting metrics for query %s",
query['query'])
continue
@ -201,7 +205,7 @@ def main():
LOG.exception("Timeout while collecting metrics for query %s",
query['query'])
continue
except pyelasticsearch.exceptions.ElasticHttpError as ex:
except elasticsearch.TransportError as ex:
LOG.error('Error from elasticsearch query for bug %s: %s',
query['bug'], ex)
continue

12
elastic_recheck/cmd/uncategorized_fails.py
View File

@ -102,7 +102,7 @@ def all_fails(classifier, config=None):
other_fails = {}
all_fails = {}
results = classifier.hits_by_query(config.all_fails_query,
size=config.uncat_search_size)
size=config.uncat_search_size, days=7)
facets = er_results.FacetSet()
facets.detect_facets(results, ["build_uuid"])
for build in facets:
@ -119,6 +119,7 @@ def all_fails(classifier, config=None):
'openstack/nova',
'openstack/requirements',
'openstack/tempest',
'openstack/tripleo-ci',
'openstack-dev/devstack',
'openstack-dev/grenade',
'openstack-infra/devstack-gate',
@ -130,6 +131,8 @@ def all_fails(classifier, config=None):
log = result.log_url.split('console.html')[0]
elif 'job-output.txt' in result.log_url:
log = result.log_url.split('job-output.txt')[0]
else:
log = '/'.join(result.log_url.split('/')[:-1])
integrated_fails["%s.%s" % (build, name)] = {
'log': log,
'timestamp': timestamp,
@ -145,6 +148,8 @@ def all_fails(classifier, config=None):
log = result.log_url.split('console.html')[0]
elif 'job-output.txt' in result.log_url:
log = result.log_url.split('job-output.txt')[0]
else:
log = ('/').join(result.log_url.split('/')[:-1])
other_fails["%s.%s" % (build, name)] = {
'log': log,
'timestamp': timestamp,
@ -215,7 +220,7 @@ def classifying_rate(fails, data, engine, classifier, ls_url):
logstash_url = ('%s/#/dashboard/file/logstash.json?%s'
% (ls_url, logstash_query))
LOG.debug("looking up hits for job %s query %s", job, query)
results = classifier.hits_by_query(query, size=1)
results = classifier.hits_by_query(query, size=1, days=7)
if results:
url['crm114'] = logstash_url
LOG.debug("Hits found. Using logstash url %s",
@ -316,7 +321,8 @@ def collect_metrics(classifier, fails, config=None):
for q in classifier.queries:
try:
results = classifier.hits_by_query(q['query'],
size=config.uncat_search_size)
size=config.uncat_search_size,
days=7)
hits = _status_count(results)
LOG.debug("Collected metrics for query %s, hits %s", q['query'],
hits)

49
elastic_recheck/config.py
View File

@ -15,20 +15,33 @@
import os
import re
import configparser
import codecs
# Can be overriden by defining environment variables with same name
DEFAULTS = {
'ES_URL': 'http://logstash.openstack.org:80/elasticsearch',
'LS_URL': 'http://logstash.openstack.org',
'ES_URL': codecs.decode(
'uggcf://xvonan:on5r4np6-624n-49sr-956r-48no8poso2o6@erivrj.' +
'eqbcebwrpg.bet/rynfgvpfrnepu/',
'rot_13'),
'LS_URL': codecs.decode(
'uggcf://xvonan:on5r4np6-624n-49sr-956r-48no8poso2o6@erivrj.' +
'eqbcebwrpg.bet/rynfgvpfrnepu/',
'rot_13'),
'DB_URI': 'mysql+pymysql://query:query@logstash.openstack.org/subunit2sql',
'server_password': '',
'CI_USERNAME': 'jenkins',
'JOBS_RE': 'dsvm',
'JOBS_RE': '(dsvm|tripleo|tox)',
'PID_FN': '/var/run/elastic-recheck/elastic-recheck.pid',
'INDEX_FORMAT': r'logstash-%Y.%m.%d',
'GERRIT_QUERY_FILE': 'queries',
'GERRIT_HOST': 'review.opendev.org',
'IRC_LOG_CONFIG': None
'GERRIT_HOST': 'review.rdoproject.org',
'GERRIT_USER': None,
'IRC_LOG_CONFIG': '',
'IRC_SERVER': "irc.oftc.net",
'IRC_PORT': "6667",
'IRC_PASS': "",
'IRC_SERVER_PASSWORD': "erbot",
'IRC_NICK': "erbot",
}
# Not all teams actively used elastic recheck for categorizing their
@ -53,16 +66,16 @@ INCLUDED_PROJECTS_REGEX = "(^openstack/|devstack|grenade)"
# Let's value legibility over pep8 line width here...
ALL_FAILS_QUERY = (
'('
'(filename:"job-output.txt" AND message:"POST-RUN END" AND message:"playbooks/base/post.yaml")' # noqa E501
'(filename:"job-output.txt" AND message:"POST-RUN END" AND message:"post.yaml")' # noqa E501
' OR '
'(filename:"console.html" AND (message:"[Zuul] Job complete" OR message:"[SCP] Copying console log" OR message:"Grabbing consoleLog"))' # noqa E501
')'
' AND build_status:"FAILURE"'
' AND build_queue:"gate"'
' AND build_queue:"check"'
' AND voting:"1"'
)
UNCAT_MAX_SEARCH_SIZE = 30000
UNCAT_MAX_SEARCH_SIZE = 10000
class Config(object):
@ -96,6 +109,11 @@ class Config(object):
self.es_index_format = es_index_format or DEFAULTS['INDEX_FORMAT']
self.pid_fn = pid_fn or DEFAULTS['PID_FN']
self.ircbot_channel_config = None
self.ircbot_server = DEFAULTS['IRC_SERVER']
self.ircbot_server_password = DEFAULTS['IRC_SERVER_PASSWORD']
self.ircbot_pass = DEFAULTS['IRC_PASS']
self.ircbot_nick = DEFAULTS['IRC_NICK']
self.ircbot_port = DEFAULTS['IRC_PORT']
self.irc_log_config = DEFAULTS['IRC_LOG_CONFIG']
self.all_fails_query = all_fails_query or ALL_FAILS_QUERY
self.excluded_jobs_regex = excluded_jobs_regex or EXCLUDED_JOBS_REGEX
@ -104,8 +122,8 @@ class Config(object):
self.uncat_search_size = uncat_search_size or UNCAT_MAX_SEARCH_SIZE
self.gerrit_query_file = (gerrit_query_file or
DEFAULTS['GERRIT_QUERY_FILE'])
self.gerrit_user = None
self.gerrit_host = None
self.gerrit_user = DEFAULTS['GERRIT_USER']
self.gerrit_host = DEFAULTS['GERRIT_HOST']
self.gerrit_host_key = None
if config_file or config_obj:
@ -121,11 +139,15 @@ class Config(object):
'gerrit_host_key': ('gerrit', 'key'),
'gerrit_query_file': ('gerrit', 'query_file'),
'gerrit_user': ('gerrit', 'user'),
'gerrit_attempts': ('gerrit', 'attempts'),
'index_format': ('data_source', 'index_format'),
'irc_log_config': ('ircbot', 'log_config'),
'ircbot_channel_config': ('ircbot', 'channel_config'),
'ircbot_server': ('ircbot', 'server_password'),
'ircbot_sever_password': ('ircbot', 'port'),
'ircbot_server': ('ircbot', 'server'),
'ircbot_server_password': ('ircbot', 'server_password'),
'ircbot_nick': ('ircbot', 'nick'),
'ircbot_pass': ('ircbot', 'pass'),
'ircbot_sever_port': ('ircbot', 'port'),
'jobs_re': ('recheckwatch', 'jobs_re'),
'ls_url': ('data_source', 'ls_url'),
'nick': ('ircbot', 'nick'),
@ -141,3 +163,6 @@ class Config(object):
configparser.NoOptionError,
configparser.NoSectionError):
pass
if self.gerrit_host_key:
self.gerrit_host_key = os.path.expanduser(self.gerrit_host_key)

77
elastic_recheck/elasticRecheck.py
View File

@ -19,7 +19,8 @@ import time
import dateutil.parser as dp
import gerritlib.gerrit
import pyelasticsearch
import elasticsearch
import requests
import sqlalchemy
from sqlalchemy import orm
from subunit2sql.db import api as db_api
@ -31,30 +32,8 @@ from elastic_recheck import results
def required_files(job):
files = []
if re.match("(tempest|grenade)-dsvm", job):
files.extend([
'logs/screen-n-api.txt',
'logs/screen-n-cpu.txt',
'logs/screen-n-sch.txt',
'logs/screen-g-api.txt',
'logs/screen-c-api.txt',
'logs/screen-c-vol.txt',
'logs/syslog.txt'])
# we could probably add more neutron files
# but currently only q-svc is used in queries
if re.match("neutron", job):
files.extend([
'logs/screen-q-svc.txt',
])
else:
files.extend([
'logs/screen-n-net.txt',
])
# make sure that grenade logs exist
if re.match("grenade", job):
files.extend(['logs/grenade.sh.txt'])
files = ["job-output.txt"]
# Can add more files for specific jobs here
return files
@ -130,7 +109,8 @@ class FailEvent(object):
bugs = self.get_all_bugs()
if not bugs:
return None
urls = ['https://bugs.launchpad.net/bugs/%s' % x for
# Return health dashboard link as all of our queries may not have bug
urls = ['http://ci-health-rdo.tripleo.org/#%s' % x for
x in bugs]
return urls
@ -168,7 +148,9 @@ class FailEvent(object):
# Assume one queue per gerrit event
if len(self.failed_jobs) == 0:
return None
return self.failed_jobs[0].url.split('/')[6]
if len(self.failed_jobs[0].url.split('/')) >= 7:
return self.failed_jobs[0].url.split('/')[6]
return None
def build_short_uuids(self):
return [job.build_short_uuid for job in self.failed_jobs]
@ -225,7 +207,7 @@ class Stream(object):
# these items. It's orthoginal to non voting ES searching.
if " (non-voting)" in line:
continue
m = re.search(r"- ([\w-]+)\s*(http://\S+)\s*:\s*FAILURE", line)
m = re.search(r"([\w-]+)\s*(https?://\S+)\s*:\s*FAILURE", line)
if m:
failed_tests.append(FailJob(m.group(1), m.group(2)))
return failed_tests
@ -243,7 +225,7 @@ class Stream(object):
def _has_required_files(self, change, patch, name, build_short_uuid):
query = qb.files_ready(change, patch, name, build_short_uuid)
r = self.es.search(query, size='80', recent=True)
files = [x['term'] for x in r.terms]
files = [x["_source"]["filename"] for x in r.hits["hits"]]
# TODO(dmsimard): Reliably differentiate zuul v2 and v3 jobs
required = required_files(name)
missing_files = [x for x in required if x not in files]
@ -255,11 +237,12 @@ class Stream(object):
def _does_es_have_data(self, event):
"""Wait till ElasticSearch is ready, but return False if timeout."""
# We wait 20 minutes wall time since receiving the event until we
# treat the logs as missing
timeout = 1200
# Wait 40 seconds between queries.
sleep_time = 40
# We wait 5 minutes wall time since receiving the event until we
# treat the logs as missing. And for now 5 minutes is enough since we
# don't have too many jobs being collected.
timeout = 300
# Wait 300 seconds between queries.
sleep_time = 300
timed_out = False
job = None
# This checks that we've got the console log uploaded, need to retry
@ -285,7 +268,7 @@ class Stream(object):
self.log.debug(e)
except FilesNotReady as e:
self.log.info(e)
except pyelasticsearch.exceptions.InvalidJsonResponseError:
except elasticsearch.SerializationError:
# If ElasticSearch returns an error code, sleep and retry
# TODO(jogo): if this works pull out search into a helper
# function that does this.
@ -329,6 +312,7 @@ class Stream(object):
# bail if the failure is from a project
# that hasn't run any of the included jobs
if not fevent.is_included_job():
self.log.debug("Ignored comment: %s", fevent.comment)
continue
self.log.info("Looking for failures in %d,%d on %s",
@ -390,7 +374,7 @@ class Classifier(object):
def most_recent(self):
"""Return the datetime of the most recently indexed event."""
query = qb.most_recent_event()
results = self.es.search(query, size='1')
results = self.es.search(query, size='1', days=7)
if len(results) > 0:
last = dp.parse(results[0].timestamp)
return last
@ -405,26 +389,27 @@ class Classifier(object):
bug_matches = []
engine = sqlalchemy.create_engine(self.config.db_uri)
Session = orm.sessionmaker(bind=engine)
session = Session()
Session()
for x in self.queries:
if x.get('suppress-notification'):
continue
self.log.debug(
"Looking for bug: https://bugs.launchpad.net/bugs/%s",
x['bug'])
query = qb.single_patch(x['query'], change_number, patch_number,
build_short_uuid)
results = self.es.search(query, size='10', recent=recent)
if len(results) > 0:
if x.get('test_ids', None):
test_ids = x['test_ids']
self.log.debug(
"For bug %s checking subunit2sql for failures on "
"test_ids: %s", x['bug'], test_ids)
if check_failed_test_ids_for_job(build_short_uuid,
test_ids, session):
bug_matches.append(x['bug'])
response = requests.get("https://bugs.launchpad.net/bugs/" +
x['bug'])
if response.status_code != 200:
bug_matches.append(x['bug'] +
": " +
x.get('msg',
re.escape(x.get('query', ""))))
else:
bug_matches.append(x['bug'])
bug_matches.append(x['bug'] + ": " + x.get(
'msg', re.escape(x.get('query', ""))))
return bug_matches

2
elastic_recheck/loader.py
View File

@ -22,7 +22,7 @@ import os.path
import yaml
def load(directory='queries'):
def load(directory="/opt/elastic-recheck/queries"):
"""Load queries from a set of yaml files in a directory."""
bugs = glob.glob("%s/*.yaml" % directory)
data = []

14
elastic_recheck/query_builder.py
View File

@ -53,7 +53,8 @@ def generic(raw_query, facet=None):
if isinstance(facet, list):
data = dict(fields=facet, size=200)
query['facets'] = {
# facets moved to aggs
query['aggs'] = {
"tag": {
"terms": data
}
@ -77,15 +78,16 @@ def result_ready(change, patchset, name, short_uuid):
"""
# TODO(dmsimard): Revisit this query once Zuul v2 is no longer supported
# Let's value legibility over pep8 line width here...
# build_short_uuid doesnt return the whole uuid in rdo es
query = (
'((filename:"job-output.txt" AND message:"POST-RUN END" AND message:"project-config/playbooks/base/post-ssh.yaml")' # noqa E501
'((filename:"job-output.txt" AND message:"POST-RUN END" AND message:"post.yaml")' # noqa E501
' OR '
'(filename:"console.html" AND (message:"[Zuul] Job complete" OR message:"[SCP] Copying console log" OR message:"Grabbing consoleLog")))' # noqa E501
' AND build_status:"FAILURE"'
' AND build_change:"{change}"'
' AND build_patchset:"{patchset}"'
' AND build_name:"{name}"'
' AND build_short_uuid:"{short_uuid}"'
' AND build_uuid:"{short_uuid}"'
)
return generic(query.format(
change=change,
@ -107,9 +109,9 @@ def files_ready(review, patch, name, build_short_uuid):
'AND build_change:"%s" '
'AND build_patchset:"%s" '
'AND build_name:"%s" '
'AND build_short_uuid:%s' %
'AND build_uuid:%s' %
(review, patch, name, build_short_uuid),
facet='filename')
facet='filename.name')
def single_patch(query, review, patch, build_short_uuid):
@ -121,7 +123,7 @@ def single_patch(query, review, patch, build_short_uuid):
return generic('%s '
'AND build_change:"%s" '
'AND build_patchset:"%s" '
'AND build_short_uuid:%s' %
'AND build_uuid:%s' %
(query, review, patch, build_short_uuid))

24
elastic_recheck/results.py
View File

@ -20,7 +20,8 @@ import datetime
import pprint
import dateutil.parser as dp
import pyelasticsearch
import elasticsearch
from elasticsearch import Elasticsearch
import pytz
@ -39,10 +40,11 @@ class SearchEngine(object):
return self.index_cache[index]
try:
es.status(index=index)
es.indices.stats(index=index)
# es.indices.status(index=index)
self.index_cache[index] = True
return True
except pyelasticsearch.exceptions.ElasticHttpNotFoundError:
except elasticsearch.exceptions.NotFoundError:
return False
def search(self, query, size=1000, recent=False, days=0):
@ -65,8 +67,9 @@ class SearchEngine(object):
The returned result is a ResultSet query.
"""
es = pyelasticsearch.ElasticSearch(self._url)
es = Elasticsearch(self._url)
args = {'size': size}
indexes = []
if recent or days:
# today's index
datefmt = self._indexfmt
@ -87,8 +90,15 @@ class SearchEngine(object):
if self._is_valid_index(es, index_name):
indexes.append(index_name)
args['index'] = indexes
results = es.search(query, **args)
if isinstance(query, str):
query = {"query": {
"query_string": {
"query": query
}
}
}
params = {"size": size, "request_timeout": 40}
results = es.search(index=indexes, body=query, params=params)
return ResultSet(results)
@ -161,7 +171,7 @@ class FacetSet(dict):
# is too large and ES won't return it. At some point we should probably
# log a warning/error for these so we can clean them up.
if facet == "timestamp" and data is not None:
ts = dp.parse(data)
ts = dp.parse(data).replace(tzinfo=pytz.utc)
tsepoch = int(calendar.timegm(ts.timetuple()))
# take the floor based on resolution
ts -= datetime.timedelta(

2
elastic_recheck/tests/functional/test_queries.py
View File

@ -67,7 +67,7 @@ class Context():
def _is_valid_ElasticSearch_query(self, x, bug) -> bool:
query = qb.generic(x['query'])
results = self.classifier.es.search(query, size='10')
results = self.classifier.es.search(query, size='10', days=1)
valid_query = len(results) > 0
if not valid_query:

2
elastic_recheck/tests/unit/__init__.py
View File

@ -52,5 +52,5 @@ class UnitTestCase(elastic_recheck.tests.TestCase):
def setUp(self):
super(UnitTestCase, self).setUp()
self.useFixture(fixtures.MonkeyPatch('pyelasticsearch.ElasticSearch',
self.useFixture(fixtures.MonkeyPatch('elasticsearch.ElasticSearch',
FakeES))

2
elastic_recheck/tests/unit/test_bot.py
View File

@ -115,7 +115,7 @@ class TestBotWithTestTools(tests.TestCase):
reference = ("openstack/keystone change: https://review.opendev.org/"
"64750 failed because of: "
"gate-keystone-python26: "
"https://bugs.launchpad.net/bugs/123456, "
"http://ci-health-rdo.tripleo.org/#123456, "
"gate-keystone-python27: unrecognized error")
self.assertEqual(reference, msg)

43
elastic_recheck/tests/unit/test_elastic_recheck.py
View File

@ -21,18 +21,25 @@ from elastic_recheck.tests import unit
class TestElasticRecheck(unit.UnitTestCase):
def test_hits_by_query_no_results(self):
c = er.Classifier("queries.yaml")
results = c.hits_by_query("this should find no bugs")
results = c.hits_by_query("this_should_find_no_bugs", days=10)
self.assertEqual(len(results), 0)
self.assertEqual(results.took, 53)
# removing took which was hardcoded to 53 as it varies
self.assertEqual(results.timed_out, False)
def test_hits_by_query(self):
# TODO(dasm): This test queries Kibana server,
# which might be unavailable at any time.
# We need to make it independent from it.
c = er.Classifier("queries.yaml")
q = ('''message:"Cannot ''createImage''"'''
''' AND filename:"console.html" AND voting:1''')
results = c.hits_by_query(q)
self.assertEqual(len(results), 20)
self.assertEqual(results.took, 46)
# updating the query to ensure we get at least some hits
q = 'filename:"job-output.txt" AND ' \
'message:"POST-RUN END" AND message:"post.yaml"'
# NOTE(dasm): Retrieve 10 days worth of upstream logs
# Otherwise, we might not have enough data in ES
results = c.hits_by_query(q, days=10)
# As 100 is the maximum results retrieved from the server
self.assertEqual(len(results), 100)
self.assertEqual(results.timed_out, False)
@ -55,25 +62,3 @@ class TestSubunit2sqlCrossover(unit.UnitTestCase):
['test1', 'test4'],
mock.sentinel.session)
self.assertFalse(res)
@mock.patch.object(er, 'check_failed_test_ids_for_job', return_value=True)
def test_classify_with_test_id_filter_match(self, mock_id_check):
c = er.Classifier('./elastic_recheck/tests/unit/queries_with_filters')
es_mock = mock.patch.object(c.es, 'search', return_value=[1, 2, 3])
es_mock.start()
self.addCleanup(es_mock.stop)
res = c.classify(1234, 1, 'fake')
self.assertEqual(res, ['1234567'],
"classify() returned %s when it should have returned "
"a list with one bug id: '1234567'" % res)
@mock.patch.object(er, 'check_failed_test_ids_for_job', return_value=False)
def test_classify_with_test_id_filter_no_match(self, mock_id_check):
c = er.Classifier('./elastic_recheck/tests/unit/queries_with_filters')
es_mock = mock.patch.object(c.es, 'search', return_value=[1, 2, 3])
es_mock.start()
self.addCleanup(es_mock.stop)
res = c.classify(1234, 1, 'fake')
self.assertEqual(res, [],
"classify() returned bug matches %s when none should "
"have been found" % res)

43
elastic_recheck/tests/unit/test_results.py
View File

@ -16,7 +16,8 @@ import datetime
import json
import mock
import pyelasticsearch
import elasticsearch
from elasticsearch import Elasticsearch
from elastic_recheck import results
from elastic_recheck import tests
@ -112,7 +113,7 @@ class MockDatetimeYesterday(datetime.datetime):
'%Y-%m-%dT%H:%M:%S')
@mock.patch.object(pyelasticsearch.ElasticSearch, 'search', return_value={})
@mock.patch.object(Elasticsearch, 'search', return_value={})
class TestSearchEngine(tests.TestCase):
"""Tests that the elastic search API is called correctly."""
@ -125,7 +126,9 @@ class TestSearchEngine(tests.TestCase):
# Tests a basic search with recent=False.
result_set = self.engine.search(self.query, size=10)
self.assertEqual(0, len(result_set))
search_mock.assert_called_once_with(self.query, size=10)
search_mock.assert_called_once_with(body={'query': {
'query_string': {'query': self.query}
}}, params={'size': 10, "request_timeout": 40}, index=[])
def _test_search_recent(self, search_mock, datetime_mock,
expected_indexes):
@ -133,14 +136,17 @@ class TestSearchEngine(tests.TestCase):
result_set = self.engine.search(self.query, size=10, recent=True)
self.assertEqual(0, len(result_set))
search_mock.assert_called_once_with(
self.query, size=10, index=expected_indexes)
body={'query': {'query_string': {'query': self.query}}},
params={'size': 10, "request_timeout": 40},
index=expected_indexes)
def test_search_recent_current_index_only(self, search_mock):
# The search index comparison goes back one hour and cuts off by day,
# so test that we're one hour and one second into today so we only have
# one index in the search call.
with mock.patch.object(
pyelasticsearch.ElasticSearch, 'status') as mock_data:
elasticsearch.client.indices.IndicesClient, 'stats') \
as mock_data:
mock_data.return_value = "Not an exception"
self._test_search_recent(search_mock, MockDatetimeToday,
expected_indexes=['logstash-2014.06.12'])
@ -150,7 +156,8 @@ class TestSearchEngine(tests.TestCase):
# so test that we're 59 minutes and 59 seconds into today so that we
# have an index for today and yesterday in the search call.
with mock.patch.object(
pyelasticsearch.ElasticSearch, 'status') as mock_data:
elasticsearch.client.indices.IndicesClient, 'stats') \
as mock_data:
mock_data.return_value = "Not an exception"
self._test_search_recent(search_mock, MockDatetimeYesterday,
expected_indexes=['logstash-2014.06.12',
@ -159,22 +166,30 @@ class TestSearchEngine(tests.TestCase):
def test_search_no_indexes(self, search_mock):
# Test when no indexes are valid
with mock.patch.object(
pyelasticsearch.ElasticSearch, 'status') as mock_data:
mock_data.side_effect = pyelasticsearch.exceptions.\
ElasticHttpNotFoundError()
elasticsearch.client.indices.IndicesClient, 'stats') \
as mock_data:
mock_data.side_effect = elasticsearch.exceptions.NotFoundError
self._test_search_recent(search_mock, MockDatetimeYesterday,
expected_indexes=[])
def test_search_days(self, search_mock):
# Test when specific days are used.
with mock.patch.object(
pyelasticsearch.ElasticSearch, 'status') as mock_data:
elasticsearch.client.indices.IndicesClient, 'stats') \
as mock_data:
mock_data.return_value = "Not an exception"
datetime.datetime = MockDatetimeYesterday
result_set = self.engine.search(self.query, size=10, days=3,
recent=False)
self.assertEqual(0, len(result_set))
search_mock.assert_called_once_with(self.query, size=10,
index=['logstash-2014.06.12',
'logstash-2014.06.11',
'logstash-2014.06.10'])
search_mock.assert_called_once_with(body={
'query': {
'query_string': {
'query': self.query
}
}
},
params={'size': 10, "request_timeout": 40},
index=['logstash-2014.06.12',
'logstash-2014.06.11',
'logstash-2014.06.10'])

8
elastic_recheck/tests/unit/test_stream.py
View File

@ -152,10 +152,10 @@ class TestStream(tests.TestCase):
self.assertTrue(event.is_included_job())
self.assertEqual(event.queue(), "gate")
self.assertEqual(event.bug_urls(),
['https://bugs.launchpad.net/bugs/123456'])
['http://ci-health-rdo.tripleo.org/#123456'])
errors = ['gate-keystone-python27: unrecognized error',
'gate-keystone-python26: '
'https://bugs.launchpad.net/bugs/123456']
'http://ci-health-rdo.tripleo.org/#123456']
bug_map = event.bug_urls_map()
for error in errors:
self.assertIn(error, bug_map)
@ -180,10 +180,10 @@ class TestStream(tests.TestCase):
self.assertTrue(event.is_included_job())
self.assertEqual(event.queue(), "check")
self.assertEqual(event.bug_urls(),
['https://bugs.launchpad.net/bugs/123456'])
['http://ci-health-rdo.tripleo.org/#123456'])
self.assertEqual(event.bug_urls_map(),
['gate-keystone-python26: '
'https://bugs.launchpad.net/bugs/123456',
'http://ci-health-rdo.tripleo.org/#123456',
'gate-keystone-python27: unrecognized error'])
self.assertEqual(sorted(event.failed_job_names()),
['gate-keystone-python26',

3
queries2/ansible-errors.yaml Normal file
View File

@ -0,0 +1,3 @@
query: >
message:"AnsibleUndefinedVariable"
AND (tags:"console.html" OR tags:"job-output.txt")

6
recheckwatchbot.yaml
View File

@ -23,14 +23,14 @@ messages:
%(bugs)s
footer: >-
For more details on this and other bugs, please see
http://status.openstack.org/elastic-recheck/
http://ci-health-rdo.tripleo.org/
recheck_instructions: >-
If you believe we've correctly identified the failure, feel free to leave a 'recheck'
comment to run the tests again.
unrecognized: >-
Some of the tests failed in a way that we did not understand. Please help
us classify these issues so that they can be part of Elastic Recheck
http://status.openstack.org/elastic-recheck/
http://ci-health-rdo.tripleo.org/
no_bugs_found: >-
I noticed Zuul failed, refer to:
http://docs.openstack.org/infra/manual/developers.html#automated-testing
http://docs.openstack.org/infra/manual/developers.html#automated-testing

2
requirements.txt
View File

@ -1,7 +1,7 @@
pbr>=1.8
python-dateutil>=2.0
pytz
pyelasticsearch<1.0
elasticsearch==7.14.0
gerritlib
python-daemon>=2.2.0
irc>=17.0

34
tools/ssh-check.py Normal file
View File

@ -0,0 +1,34 @@
import logging
import paramiko
import os
logging.basicConfig()
LOG = logging.getLogger("paramiko")
LOG.setLevel(logging.DEBUG)
hostname = "review.opendev.org"
port = 29418
username = os.environ.get("GERRIT_USER", None) # current user unless mentioned
keyfile = None # implicit key, if any
LOG.info(f"Trying ssh connection to {username}@{hostname}:{port}")
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(
hostname,
username=username,
port=port,
key_filename=keyfile)
(stdin, stdout, stderr) = client.exec_command("gerrit version")
for line in stdout.readlines():
print(line)
# Avoid confusing "TypeError: 'NoneType' object is not callable" exception
# https://github.com/paramiko/paramiko/issues/1078
if client is not None:
client.close()
del client, stdin, stdout, stderr

2
tools/unaccounted_rechecks.py
View File

@ -28,7 +28,7 @@ def get_options():
description='Find rechecks not accounted for in ER')
parser.add_argument('-u', '--user', help='Gerrit User',
default=getpass.getuser())
tryfiles = ('id_gerrit', 'id_rsa', 'id_dsa')
tryfiles = ('id_gerrit', 'id_rsa', 'id_ecdsa', 'id_dsa')
default_key = ""
for f in tryfiles:
trykey = os.path.join(os.path.expanduser("~"), '.ssh', f)

4
tox.ini
View File

@ -58,8 +58,8 @@ commands = bindep test
[testenv:linters]
basepython = python3
deps =
flake8==3.8.3
pylint==2.6.0
flake8==4.0.1
pylint==2.8.3
-r{toxinidir}/test-requirements.txt
commands =
flake8

27
web/conf/nginx.conf Normal file
View File

@ -0,0 +1,27 @@
# /etc/nginx/conf.d/default.conf
server {
listen 80 default_server;
listen [::]:80 default_server;
error_log /dev/stdout info;
root /var/www/localhost;
location ~ ^/data/(.*) {
alias /data/www/$1;
}
location ~ ^/elastic-recheck\/data/(.*\.json) {
alias /data/www/$1;
}
location / {
error_page 404 = @fallback;
}
location @fallback {
proxy_pass http://status.openstack.org;
}
}

BIN
web/share/favicon.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 15 KiB

47
web/share/gate.html
View File

@ -1,58 +1,58 @@
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:py="http://genshi.edgewall.org/"
<html xmlns="https://www.w3.org/1999/xhtml"
xmlns:py="https://genshi.edgewall.org/"
lang="en">
<HEAD>
<TITLE>Elastic Recheck</TITLE>
<script type="text/javascript"
src="http://status.openstack.org/jquery.min.js"></script>
src="/jquery.min.js"></script>
<script type="text/javascript" src="handlebars-v2.0.0.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery-visibility.min.js"></script>
src="/jquery-visibility.min.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery-graphite.js"></script>
src="/jquery-graphite.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/common.js"></script>
src="/common.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.canvaswrapper.js"></script>
src="/jquery.canvaswrapper.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.js"></script>
src="/jquery.flot.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.saturated.js"></script>
src="/jquery.flot.saturated.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.uiConstants.js"></script>
src="/jquery.flot.uiConstants.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.browser.js"></script>
src="/jquery.flot.browser.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.colorhelpers.js"></script>
src="/jquery.colorhelpers.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.drawSeries.js"></script>
src="/jquery.flot.drawSeries.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.time.js"></script>
src="/jquery.flot.time.js"></script>
<script type="text/javascript"
src="elastic-recheck.js"></script>
src="/elastic-recheck.js"></script>
<script type='text/javascript'>
var data_url = '/elastic-recheck/data/gate.json';
</script>
<!-- Google Fonts -->
<link href='http://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>
<link href='https://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>
<!-- Framework CSS -->
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>
<link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
<link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>
<!-- IE CSS -->
<!--[if lt IE 8]><link rel="stylesheet" href="http://www.openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->
<!--[if lt IE 8]><link rel="stylesheet" href="https://www.openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->
<!-- OpenStack Specific CSS -->
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>
<link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>
<!-- Page Specific CSS -->
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>
<link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>
<link rel="stylesheet" type="text/css" href="http://www.openstack.org/themes/openstack/css/main.css" />
<link rel="stylesheet" type="text/css" href="https://www.openstack.org/themes/openstack/css/main.css" />
<!-- Project specific css -->
@ -73,6 +73,7 @@
<h3>{{bug.fails24}} fails in 24 hrs / {{bug.fails}} fails in
10 days</h3>
<h3>Projects: {{bug.bug_data.affects}}</h3>
<h3>Details: {{bug.msg}}</h3>
<!-- only display that a job is non-voting since voting:1 is the
default filter -->
{{#unless bug.voting}}
@ -88,7 +89,7 @@
{{/if}}
<div class="graph"></div>
<a class="extlink" href="{{bug.logstash_url}}">Logstash</a>
<a class="extlink" href="http://bugs.launchpad.net/bugs/{{bug.number}}">Launchpad</a>
<a class="extlink" href="https://bugs.launchpad.net/bugs/{{bug.number}}">Launchpad</a>
</div>
</script>

49
web/share/index.html
View File

@ -1,59 +1,59 @@
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:py="http://genshi.edgewall.org/"
<html xmlns="https://www.w3.org/1999/xhtml"
xmlns:py="https://genshi.edgewall.org/"
lang="en">
<HEAD>
<TITLE>Elastic Recheck</TITLE>
<script type="text/javascript"
src="http://status.openstack.org/jquery.min.js"></script>
src="/jquery.min.js"></script>
<script type="text/javascript" src="handlebars-v2.0.0.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery-visibility.min.js"></script>
src="/jquery-visibility.min.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery-graphite.js"></script>
src="/jquery-graphite.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/common.js"></script>
src="/common.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.canvaswrapper.js"></script>
src="/jquery.canvaswrapper.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.js"></script>
src="/jquery.flot.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.saturated.js"></script>
src="/jquery.flot.saturated.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.uiConstants.js"></script>
src="/jquery.flot.uiConstants.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.browser.js"></script>
src="/jquery.flot.browser.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.colorhelpers.js"></script>
src="/jquery.colorhelpers.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.drawSeries.js"></script>
src="/jquery.flot.drawSeries.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.time.js"></script>
src="/jquery.flot.time.js"></script>
<script type="text/javascript"
src="elastic-recheck.js"></script>
src="/elastic-recheck.js"></script>
<script type='text/javascript'>
var data_url = '/elastic-recheck/data/all.json';
</script>
<!-- Google Fonts -->
<link href='http://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>
<link href='https://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>
<!-- Framework CSS -->
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>
<link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
<link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>
<!-- IE CSS -->
<!--[if lt IE 8]><link rel="stylesheet" href="http://www.openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->
<!--[if lt IE 8]><link rel="stylesheet" href="https://www.openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->
<!-- OpenStack Specific CSS -->
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>
<link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>
<!-- Page Specific CSS -->
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>
<link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>
<link rel="stylesheet" type="text/css"
href="http://www.openstack.org/themes/openstack/css/main.css" />
href="https://www.openstack.org/themes/openstack/css/main.css" />
@ -76,6 +76,7 @@
<h3>{{bug.fails24}} fails in 24 hrs / {{bug.fails}} fails in
10 days</h3>
<h3>Projects: {{bug.bug_data.affects}}</h3>
<h3>Details: {{bug.msg}}</h3>
{{#if bug.bug_data.reviews}}
<h3 class="openreviews">Open Reviews:</h3>
<div class="openreviews">
@ -86,7 +87,7 @@
{{/if}}
<div class="graph"></div>
<a class="extlink" href="{{bug.logstash_url}}">Logstash</a>
<a class="extlink" href="http://bugs.launchpad.net/bugs/{{bug.number}}">Launchpad</a>
<a class="extlink" href="https://bugs.launchpad.net/bugs/{{bug.number}}">Launchpad</a>
</div>
</script>
<div class="container">
@ -96,7 +97,7 @@
<li><a href="data/integrated_gate.html">Uncategorized Integrated Gate Jobs</a></li>
<li><a href="data/others.html">Uncategorized</a></li>
</ul>
<p>The elastic-recheck project uses Elasticsearch to classify and track OpenStack gate failures. Documentation can be found here: <a href="http://docs.openstack.org/infra/elastic-recheck/">http://docs.openstack.org/infra/elastic-recheck/</a>. You can also learn more by reading this post on the Elasticsearch blog: <a href="http://www.elasticsearch.org/blog/openstack-elastic-recheck-powered-elk-stack/">OpenStack elastic-recheck: powered by the elk stack</a>.</p>
<p>The elastic-recheck project uses Elasticsearch to classify and track OpenStack gate failures. Documentation can be found here: <a href="https://docs.openstack.org/infra/elastic-recheck/">docs.openstack.org/infra/elastic-recheck/</a>. You can also learn more by reading this post on the Elasticsearch blog: <a href="https://www.elasticsearch.org/blog/openstack-elastic-recheck-powered-elk-stack/">OpenStack elastic-recheck: powered by the elk stack</a>.</p>
</div>
<div id="vital-stats" class="container">

30
web/share/templates/base.html
View File

@ -1,42 +1,42 @@
<html xmlns="http://www.w3.org/1999/xhtml"
xmlns:py="http://genshi.edgewall.org/"
<html xmlns="https://www.w3.org/1999/xhtml"
xmlns:py="https://genshi.edgewall.org/"
lang="en">
<HEAD>
<TITLE>Elastic Recheck</TITLE>
<script type="text/javascript"
src="http://status.openstack.org/jquery.min.js"></script>
src="/jquery.min.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery-visibility.min.js"></script>
src="/jquery-visibility.min.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery-graphite.js"></script>
src="/jquery-graphite.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/common.js"></script>
src="/common.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.min.js"></script>
src="/jquery.flot.js"></script>
<script type="text/javascript"
src="http://status.openstack.org/jquery.flot.time.min.js"></script>
src="/jquery.flot.time.js"></script>
<script type="text/javascript"
src="elastic-recheck.js"></script>
src="/elastic-recheck.js"></script>
<!-- Google Fonts -->
<link href='http://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>
<link href='https://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>
<!-- Framework CSS -->
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>
<link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
<link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>
<!-- IE CSS -->
<!--[if lt IE 8]><link rel="stylesheet" href="http://www.openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->
<!-- OpenStack Specific CSS -->
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>
<link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>
<!-- Page Specific CSS -->
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>
<link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>
<link rel="stylesheet" type="text/css" href="http://www.openstack.org/themes/openstack/css/main.css" />
<link rel="stylesheet" type="text/css" href="https://www.openstack.org/themes/openstack/css/main.css" />
<!-- Project specific css -->
<link rel="stylesheet" type="text/css" href="../styles/elastic-recheck.css" />