Run elastic-recheck in container

There is an API mismatch between Upstream and RDO ES. The existing code uses pyelasticsearch which will not work with newer version of ES in RDO. This has created a complexity in patch [1]. To address this we are breaking down [1] into simpler patches - Current patch containerizes ER and works with upstream ES only - Separate patch will update pyelasticsearch client and work with rdo es [1] https://review.opendev.org/c/opendev/elastic-recheck/+/729623 Story: TRIPLEOCI-188 Change-Id: I5491a034fa7718ebf9c0251449b071b88b0b713b (cherry picked from commit 6d7aca0c27)
2021-07-29 13:52:25 +02:00 · 2021-07-29 13:52:25 +02:00 · d254371944
parent 01b6b8299b
commit d254371944
29 changed files with 536 additions and 85 deletions
--- a/.gitignore
+++ b/.gitignore
@ -53,3 +53,8 @@ ChangeLog
 # Local data to test web-served html pages
 web/share/elastic-recheck
 web/share/data
+
+# logs
+data/*.log
+data/id_rsa
+data/id_rsa.pub
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -28,6 +28,7 @@

 - job:
    name: elastic-recheck-container
+    voting: false
    parent: opendev-build-docker-image
    description: Build container images for elastic-recheck service
    vars:
--- a/61
+++ b/61
@ -1,18 +1,61 @@
 # syntax=docker/dockerfile:experimental
-FROM opendevorg/python-builder:3.7 as elastic-recheck-builder
+# We make use of alpine as it seems to have a more container friendly cron
+# later alpine versions require different nginx ocnfiguration
+FROM alpine:3.13 as elastic-recheck

+# FROM opendevorg/python-builder:3.7 as elastic-recheck-builder
+
+RUN \
+    apk update && \
+    apk add \
+    git \
+    nginx \
+    py3-argparse \
+    py3-babel \
+    py3-certifi \
+    py3-cffi \
+    py3-cryptography \
+    py3-distro \
+    py3-elasticsearch \
+    py3-httplib2 \
+    py3-jinja2 \
+    py3-netaddr \
+    py3-netifaces \
+    py3-oauthlib \
+    py3-paramiko \
+    py3-pbr \
+    py3-pip \
+    py3-requests \
+    py3-simplejson \
+    py3-sqlalchemy \
+    py3-tempita \
+    py3-tz \
+    py3-yaml \
+&& ln -f -s /data/cron/crontab /etc/crontabs/root
+WORKDIR /tmp/src
 COPY . /tmp/src
-RUN assemble
+RUN pip3 install .

-FROM opendevorg/python-base:3.7 as elastic-recheck
+# RUN assemble
+# FROM opendevorg/python-base:3.7 as elastic-recheck
+# COPY --from=elastic-recheck-builder /output/ /output

-COPY --from=elastic-recheck-builder /output/ /output
-RUN /output/install-from-bindep && \
-rm -rf /output
-COPY data/  /data/
-COPY queries/ /opt/elastic-recheck/queries
+# RUN /output/install-from-bindep && \
+# rm -rf /output && \
+RUN rm -rf /tmp/src && \
+    mkdir -p /root/.ssh /data /run/nginx && \
+    chmod 700 /root/.ssh
+
+COPY web/conf/nginx.conf /etc/nginx/conf.d/default.conf
+COPY web/share/ /var/www/localhost
+COPY data/cron/  /root/
+COPY data/elastic-recheck.conf /root/elastic-recheck.conf
+COPY data/recheckwatchbot.yaml /root/recheckwatchbot.yaml
+COPY tools/ssh-check.py  /root/ssh-check.py
+# COPY data/crontab  /var/spool/cron/crontabs/root
+COPY data/id_rsa  /root/.ssh/id_rsa

 # using root allows us to use same relative paths in configs for running outside
 # containers, where ./data contains persistent configs and logs.
 WORKDIR /
-CMD /usr/local/bin/elastic-recheck -f data/elastic-recheck.conf ${ER_OPTS:-}
+CMD /usr/bin/elastic-recheck -f /root/elastic-recheck.conf ${ER_OPTS:-}
--- a/58
+++ b/58
@ -9,6 +9,9 @@ ENGINE ?= $(shell command -v docker podman|head -n1)
 IMAGE_TAG=localhost/elastic-recheck
 # Enable newer docker buildkit if available
 DOCKER_BUILDKIT=1
+COMPOSE_DOCKER_CLI_BUILD=1
+# ssh opts to add, used only for testing
+SSH_OPTS=-o "StrictHostKeyChecking no" -o "UserKnownHostsFile /dev/null"

 .PHONY: default
 default: help
@ -32,16 +35,67 @@ export PRINT_HELP_PYSCRIPT
 help:
 	@$(PYTHON) -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)

+.PHONY: check-env
+check-env:
+ifndef GERRIT_USER
+	$(error GERRIT_USER is undefined, you need to define it to run this command)
+endif
+
 .PHONY: build
-build:  ## Build image using docker
+build: data/id_rsa check-env  ## Build image using docker
+	@echo "Checking that current user can connect to gerit using ssh..."""
+	@python3 ./tools/ssh-check.py
 	$(ENGINE) build -t $(IMAGE_TAG) .
 	@echo "Image size: $$(docker image inspect --format='scale=0; {{.Size}}/1024/1024' $(IMAGE_TAG) | bc)MB"
+	@echo "Validate that built container can also connect to gerrit..."""
+	$(ENGINE) run --env GERRIT_USER -it $(IMAGE_TAG) python3 /root/ssh-check.py
+
+.PHONY: up
+up: data/id_rsa check-env ## Run containers
+	@# validates that container has credentials and connectivity to talk with gerrit server
+	@# Validate the builder image can connect to server
+	@# $(ENGINE) run --env GERRIT_USER -it $(IMAGE_TAG) python3 /root/ssh-check.py
+	@# --abort-on-container-exit
+	docker-compose up --force-recreate --remove-orphans --abort-on-container-exit <<<y
+	@# docker container stop elastic-recheck
+	@# bash -c "docker rm elastic-recheck || true"
+	@# docker run --env ES_URL --env LS_URL --env DB_URI --env GERRIT_HOST --env GERRIT_USER --env IRC_NICK --env IRC_PASS -it --name elastic-recheck elastic-recheck:latest
+
+.PHONY: down
+down: ## Destroy containers
+	docker-compose down --remove-orphans --rmi local

 .PHONY: into
 into: ## Starts bash inside docker image
-	$(ENGINE) run -it $(IMAGE_TAG) /bin/bash
+	$(ENGINE) run --mount 'type=volume,src=er-volume,dst=/data,volume-driver=local' -it $(IMAGE_TAG) /bin/sh
+	# $(ENGINE) exec -it er-cron /bin/sh
+	@# docker-compose run elastic-recheck bash

 .PHONY: dive
 dive:  ## Use `dive` tool to investigate container size
 	# https://github.com/wagoodman/dive
 	dive $(IMAGE_TAG)
+
+data/id_rsa:
+	# this key must be unencrypted, so create a spare one for testing and
+	# add it to your gerrit user configuration
+	cp -f ~/.ssh/id_rsa_insecure data/id_rsa
+
+.PHONY: clean
+clean:  ## Use clean to remove all temp files, including container and images but **not** data/
+	$(ENGINE) image rm --force $(IMAGE_TAG)
+
+.PHONY: key
+key: ## Makes a SSH key compatibile with paramiko (overrides existing one)
+	@mkdir -p data
+	@rm data/id_rsa || true
+	@ssh-keygen -q -m PEM -t rsa -b 2048 -f data/id_rsa <<<$$'\n'
+	@ssh-keygen -l -f data/id_rsa
+	@echo "WARN: Please assign key below to your gerrit user using the web interface:"
+	@ssh-keygen -l -f data/id_rsa
+
+
+.PHONY: wheel
+wheel:  ## Use clean to remove all temp files, including container and images but **not** data/
+	mkdir -p build
+	cd build && pip3 wheel ..
--- a/README.rst
+++ b/README.rst
@ -229,6 +229,23 @@ You can execute an individual query locally and analyze the search results::
      95% master
      4% stable/icehouse

+
+Container
+---------
+
+Elastic Recheck container is a newer deployment model that use a docker-compose
+file that deploys the 3 services using the same container. They all share a
+single persistent ``er-volume``.
+
+The services are:
+
+* ``er-bot`` - This is watching gerrit even stream for new events
+* ``er-cron`` - This refreshes the 3 json files on a schedule
+* ``er-web`` - Provides the dashboard frontend
+
+A developer should easily start all of these by running ``make up``. By default
+if any of the services is failing all containers will be stopped.
+
 Notes
 -----

--- a/bindep.txt
+++ b/bindep.txt
@ -16,3 +16,4 @@ libopenssl-devel [platform:suse !platform:rpm test compile]
 locales [platform:debian]
 python3-dev [platform:dpkg test compile]
 python3-devel [platform:rpm test compile]
+python3-cryptography [platform:rpm]
--- a/data/cron/cron-er-graph-all.sh
+++ b/data/cron/cron-er-graph-all.sh
@ -0,0 +1,5 @@
+#!/bin/sh
+set -ex
+cd /data/www
+/root/er-safe-run.sh elastic-recheck-graph /opt/elastic-recheck/queries -o all-new.json
+mv all-new.json all.json
--- a/data/cron/cron-er-graph-gate.sh
+++ b/data/cron/cron-er-graph-gate.sh
@ -0,0 +1,4 @@
+#!/bin/sh
+set -ex
+cd /data/www
+/root/er-safe-run.sh elastic-recheck-graph /opt/elastic-recheck/queries -o gate-new.json -q gate && mv gate-new.json gate.json
--- a/data/cron/cron-er-graph-uncat.sh
+++ b/data/cron/cron-er-graph-uncat.sh
@ -0,0 +1,6 @@
+#!/bin/sh
+set -ex
+cd /data/www
+mkdir -p new
+/root/er-safe-run.sh elastic-recheck-uncategorized -d /opt/elastic-recheck/queries -t /var/www/localhost/templates -o new
+mv new/*.html .
--- a/data/cron/cron-start.sh
+++ b/data/cron/cron-start.sh
@ -0,0 +1,13 @@
+#!/bin/sh
+set -ex
+
+# er-volume may be empty at start
+mkdir -p /data/www
+
+# Used under docker to emulate cron execution, but we force running all
+# jobs first, so we do not have to wait for refresh when newly deployed.
+/root/cron-er-graph-uncat.sh
+/root/cron-er-graph-gate.sh
+/root/cron-er-graph-all.sh
+# Run cron in foreground foreever:
+crond -f -d 2
--- a/data/cron/crontab
+++ b/data/cron/crontab
@ -0,0 +1,4 @@
+PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin
+00,30 * * * * /root/cron-er-graph-all.sh
+10,40 * * * * /root/cron-er-graph-gate.sh
+20,50 * * * * /root/cron-er-graph-uncat.sh
--- a/data/cron/er-safe-run.sh
+++ b/data/cron/er-safe-run.sh
@ -0,0 +1,5 @@
+#!/bin/sh
+
+# Timeout after 4 hours as we've seen this script occasionally fail to
+# exit after running for days.
+flock /var/run/er-safe-run.lock timeout -s KILL 14400 $@
--- a/data/elastic-recheck.conf
+++ b/data/elastic-recheck.conf
@ -0,0 +1,33 @@
+# Relative paths use container WORKDIR (/) or repo root when outside container
+# [DEFAULT]
+# GERRIT_QUERY_FILE=/opt/elastic-recheck/queries
+# IRC_NICK=elastic-recheck
+# IRC_PASS=
+# IRC_CHANNEL_CONFIG=recheckwatchbot.yaml
+# LOG_CONFIG=data/logging.config
+# JOBS_RE=(dsvm|tripleo|devstack|tempest|grenade)
+
+[ircbot]
+# nick=${IRC_NICK}
+# pass=${IRC_PASS}
+server=irc.freenode.net
+port=6667
+channel_config=recheckwatchbot.yaml
+# log_config=${LOG_CONFIG}
+
+[gerrit]
+# host=${GERRIT_HOST}
+# user=${GERRIT_USER}
+# query_file=${GERRIT_QUERY_FILE}
+# key=data/id_rsa
+
+[data_source]
+es_url=${ES_URL}
+ls_url=${LS_URL}
+db_uri=${DB_URI}
+
+[recheckwatch]
+# TODO(mriedem): With zuul v3 there is a wild west of job names so this
+# regex is table stakes, but there are clearly things missing from this like
+# the various nova-* jobs (nova-multiattach, nova-live-migration, nova-next).
+jobs_re=(dsvm|tripleo|devstack|tempest|grenade)
--- a/data/id_rsa
+++ b/data/id_rsa
@ -0,0 +1 @@
+This is a dummy file
--- a/data/logging.config
+++ b/data/logging.config
@ -0,0 +1,52 @@
+[loggers]
+keys=root,recheckwatchbot,elastic-recheck,irc
+
+[handlers]
+keys=console,debug,normal
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=DEBUG
+handlers=console,debug,normal
+
+[logger_elastic-recheck]
+level=DEBUG
+handlers=console,debug,normal
+qualname=elastic-recheck
+propagate=0
+
+[logger_recheckwatchbot]
+level=DEBUG
+handlers=console,debug,normal
+qualname=recheckwatchbot
+propagate=0
+
+[logger_irc]
+level=DEBUG
+handlers=console,debug,normal
+qualname=irc
+propagate=0
+
+[handler_console]
+level=WARNING
+class=StreamHandler
+formatter=simple
+args=(sys.stdout,)
+
+[handler_debug]
+level=DEBUG
+class=logging.handlers.TimedRotatingFileHandler
+formatter=simple
+args=('data/elastic-recheck_debug.log', 'midnight', 1, 30,)
+
+[handler_normal]
+level=INFO
+class=logging.handlers.TimedRotatingFileHandler
+formatter=simple
+args=('data/elastic-recheck.log', 'midnight', 1, 30,)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(name)s: %(message)s
+datefmt=
--- a/data/recheckwatchbot.yaml
+++ b/data/recheckwatchbot.yaml
@ -0,0 +1,38 @@
+channels:
+  openstack-neutron:
+    projects:
+      # elastic-recheck doesn't allow to limit reports to patches from a specific repo,
+      # so let's at least scope to bugs that neutron team acknowledged ownership for
+      # in Launchpad
+      - neutron
+    events:
+      # to limit reports to failures that belong to bugs owned by neutron team;
+      # we may revisit it once elastic-recheck learns how to limit the scope of reports
+      # to particular repositories
+      - positive
+  openstack-qa:
+    projects:
+      - all
+    events:
+      - positive
+      - negative
+
+messages:
+  # | means don't fold newlines, > means do
+  found_bug: |
+    I noticed Zuul failed, I think you hit bug(s):
+
+    %(bugs)s
+  footer: >-
+    For more details on this and other bugs, please see
+    http://status.openstack.org/elastic-recheck/
+  recheck_instructions: >-
+    If you believe we've correctly identified the failure, feel free to leave a 'recheck'
+    comment to run the tests again.
+  unrecognized: >-
+    Some of the tests failed in a way that we did not understand. Please help
+    us classify these issues so that they can be part of Elastic Recheck
+    http://status.openstack.org/elastic-recheck/
+  no_bugs_found: >-
+    I noticed Zuul failed, refer to:
+    https://docs.openstack.org/infra/manual/developers.html#automated-testing
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,85 @@
+version: '2'
+services:
+  cron:
+    container_name: er-cron
+    image: localhost/elastic-recheck
+    # the first time it starts we want to update as fast as possible, and
+    # switch to cron afterwords.
+    command: /root/cron-start.sh
+    environment:
+      - DB_URI
+      - ES_URL=${ES_URL}
+      - GERRIT_HOST
+      # - GERRIT_KEY
+      - GERRIT_USER=${GERRIT_USER}
+      - IRC_NICK
+      - IRC_PASS
+      - LOG_CONFIG
+      - LS_URL=${LS_URL}
+    volumes:
+      - er-volume:/data
+      # mount queries from outside the container, so we can update them w/o having to restert it
+      - ../tripleo-ci-health-queries/output/elastic-recheck:/opt/elastic-recheck/queries
+  bot:
+    container_name: er-bot
+    image: localhost/elastic-recheck
+    working_dir: /root
+    command: /usr/bin/elastic-recheck -f elastic-recheck.conf
+    environment:
+      - DB_URI
+      - ES_URL
+      - GERRIT_HOST
+      # - GERRIT_KEY
+      - GERRIT_USER
+      - IRC_NICK
+      - IRC_PASS
+      - LOG_CONFIG
+      - LS_URL
+    volumes:
+      - er-volume:/data
+  web:
+    container_name: er-web
+    image: localhost/elastic-recheck
+    command: nginx -g 'daemon off;'
+    environment:
+      - DB_URI
+      - ES_URL=${ES_URL}
+      - GERRIT_HOST
+      # - GERRIT_KEY
+      - GERRIT_USER=${GERRIT_USER}
+      - IRC_NICK
+      - IRC_PASS
+      - LOG_CONFIG
+      - LS_URL=${LS_URL}
+    ports:
+      - 80:80
+    # we do not want to start it too soon as it may fail to start if
+    # the er-volume is empty
+    depends_on:
+      - cron
+      - bot
+    volumes:
+      - er-volume:/data
+    labels:
+      - traefik.enable=true
+      - traefik.port=80
+      - traefik.http.routers.er.rule=Host(`er.sbarnea.com`)
+      - traefik.http.routers.er.tls.certResolver=myresolver
+      - traefik.http.routers.er.entrypoints=websecure
+      - traefik.http.services.er.loadbalancer.server.port=80
+
+      # do not mention network_mode or we risk getting:
+    # conflicting options: host type networking can't be used with links
+    # network_mode: host
+#     networks:
+#           hostnet: {}
+
+volumes:
+  er-volume:
+    name: er-volume
+    driver: local
+
+# # networks:
+# #   hostnet:
+# #     external: true
+# #     name: host
--- a/elastic_recheck/bot.py
+++ b/elastic_recheck/bot.py
@ -237,7 +237,7 @@ class ChannelConfig(object):
        self.data = data

        keys = data.keys()
-        for key in keys:
+        for key in list(keys):
            if key[0] != '#':
                data['#' + key] = data.pop(key)
        self.channels = data.keys()
@ -291,6 +291,7 @@ def _main(args, config):
    msgs = MessageConfig(yaml.safe_load(open(fp)))

    if not args.noirc:
+        print(dir(config))
        bot = RecheckWatchBot(
            channel_config.channels,
            config=config)
--- a/elastic_recheck/cmd/check_success.py
+++ b/elastic_recheck/cmd/check_success.py
@ -52,7 +52,8 @@ def all_fails(classifier):
    so we can figure out how good we are doing on total classification.
    """
    all_fails = {}
-    results = classifier.hits_by_query(er_config.ALL_FAILS_QUERY, size=30000)
+    results = classifier.hits_by_query(er_config.ALL_FAILS_QUERY,
+                                       size=30000, days=14)
    facets = er_results.FacetSet()
    facets.detect_facets(results, ["build_uuid"])
    for build in facets:
@ -165,7 +166,7 @@ def collect_metrics(classifier, fails):
    data = {}
    for q in classifier.queries:
        start = time.time()
-        results = classifier.hits_by_query(q['query'], size=30000)
+        results = classifier.hits_by_query(q['query'], size=30000, days=14)
        log = logging.getLogger('recheckwatchbot')
        log.debug("Took %d seconds to run (uncached) query for bug %s",
                  time.time() - start, q['bug'])
--- a/elastic_recheck/cmd/graph.py
+++ b/elastic_recheck/cmd/graph.py
@ -161,8 +161,6 @@ def main():
    }

    # Get the cluster health for the header
-    es = pyelasticsearch.ElasticSearch(config.es_url)
-    jsondata['status'] = es.health()['status']

    for query in classifier.queries:
        if args.queue:
--- a/elastic_recheck/cmd/uncategorized_fails.py
+++ b/elastic_recheck/cmd/uncategorized_fails.py
@ -102,7 +102,7 @@ def all_fails(classifier, config=None):
    other_fails = {}
    all_fails = {}
    results = classifier.hits_by_query(config.all_fails_query,
-                                       size=config.uncat_search_size)
+                                       size=config.uncat_search_size, days=14)
    facets = er_results.FacetSet()
    facets.detect_facets(results, ["build_uuid"])
    for build in facets:
@ -130,6 +130,8 @@ def all_fails(classifier, config=None):
                    log = result.log_url.split('console.html')[0]
                elif 'job-output.txt' in result.log_url:
                    log = result.log_url.split('job-output.txt')[0]
+                else:
+                    log = '/'.join(result.log_url.split('/')[:-1])
                integrated_fails["%s.%s" % (build, name)] = {
                    'log': log,
                    'timestamp': timestamp,
@ -215,7 +217,7 @@ def classifying_rate(fails, data, engine, classifier, ls_url):
            logstash_url = ('%s/#/dashboard/file/logstash.json?%s'
                            % (ls_url, logstash_query))
            LOG.debug("looking up hits for job %s query %s", job, query)
-            results = classifier.hits_by_query(query, size=1)
+            results = classifier.hits_by_query(query, size=1, days=14)
            if results:
                url['crm114'] = logstash_url
                LOG.debug("Hits found. Using logstash url %s",
--- a/elastic_recheck/config.py
+++ b/elastic_recheck/config.py
@ -28,7 +28,13 @@ DEFAULTS = {
    'INDEX_FORMAT': r'logstash-%Y.%m.%d',
    'GERRIT_QUERY_FILE': 'queries',
    'GERRIT_HOST': 'review.opendev.org',
-    'IRC_LOG_CONFIG': None
+    'GERRIT_USER': None,
+    'IRC_LOG_CONFIG': '',
+    'IRC_SERVER': "irc.freenode.net",
+    'IRC_PORT': "6667",
+    'IRC_PASS': "",
+    'IRC_SERVER_PASSWORD': "",
+    'IRC_NICK': "",
 }

 # Not all teams actively used elastic recheck for categorizing their
@ -96,6 +102,11 @@ class Config(object):
        self.es_index_format = es_index_format or DEFAULTS['INDEX_FORMAT']
        self.pid_fn = pid_fn or DEFAULTS['PID_FN']
        self.ircbot_channel_config = None
+        self.ircbot_server = DEFAULTS['IRC_SERVER']
+        self.ircbot_server_password = DEFAULTS['IRC_SERVER_PASSWORD']
+        self.ircbot_pass = DEFAULTS['IRC_PASS']
+        self.ircbot_nick = DEFAULTS['IRC_NICK']
+        self.ircbot_port = DEFAULTS['IRC_PORT']
        self.irc_log_config = DEFAULTS['IRC_LOG_CONFIG']
        self.all_fails_query = all_fails_query or ALL_FAILS_QUERY
        self.excluded_jobs_regex = excluded_jobs_regex or EXCLUDED_JOBS_REGEX
@ -104,8 +115,8 @@ class Config(object):
        self.uncat_search_size = uncat_search_size or UNCAT_MAX_SEARCH_SIZE
        self.gerrit_query_file = (gerrit_query_file or
                                  DEFAULTS['GERRIT_QUERY_FILE'])
-        self.gerrit_user = None
-        self.gerrit_host = None
+        self.gerrit_user = DEFAULTS['GERRIT_USER']
+        self.gerrit_host = DEFAULTS['GERRIT_HOST']
        self.gerrit_host_key = None

        if config_file or config_obj:
@ -121,11 +132,15 @@ class Config(object):
                'gerrit_host_key': ('gerrit', 'key'),
                'gerrit_query_file': ('gerrit', 'query_file'),
                'gerrit_user': ('gerrit', 'user'),
+                'gerrit_attempts': ('gerrit', 'attempts'),
                'index_format': ('data_source', 'index_format'),
                'irc_log_config': ('ircbot', 'log_config'),
                'ircbot_channel_config': ('ircbot', 'channel_config'),
-                'ircbot_server': ('ircbot', 'server_password'),
-                'ircbot_sever_password': ('ircbot', 'port'),
+                'ircbot_server': ('ircbot', 'server'),
+                'ircbot_server_password': ('ircbot', 'server_password'),
+                'ircbot_nick': ('ircbot', 'nick'),
+                'ircbot_pass': ('ircbot', 'pass'),
+                'ircbot_sever_port': ('ircbot', 'port'),
                'jobs_re': ('recheckwatch', 'jobs_re'),
                'ls_url': ('data_source', 'ls_url'),
                'nick': ('ircbot', 'nick'),
@ -141,3 +156,6 @@ class Config(object):
                        configparser.NoOptionError,
                        configparser.NoSectionError):
                    pass
+
+            if self.gerrit_host_key:
+                self.gerrit_host_key = os.path.expanduser(self.gerrit_host_key)
--- a/queries2/ansible-errors.yaml
+++ b/queries2/ansible-errors.yaml
@ -0,0 +1,3 @@
+query: >
+  message:"AnsibleUndefinedVariable"
+  AND (tags:"console.html" OR tags:"job-output.txt")
--- a/tools/ssh-check.py
+++ b/tools/ssh-check.py
@ -0,0 +1,34 @@
+import logging
+import paramiko
+import os
+
+
+logging.basicConfig()
+LOG = logging.getLogger("paramiko")
+LOG.setLevel(logging.DEBUG)
+
+hostname = "review.opendev.org"
+port = 29418
+username = os.environ.get("GERRIT_USER", None)  # current user unless mentioned
+keyfile = None  # implicit key, if any
+
+LOG.info(f"Trying ssh connection to {username}@{hostname}:{port}")
+client = paramiko.SSHClient()
+client.load_system_host_keys()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+client.connect(
+    hostname,
+    username=username,
+    port=port,
+    key_filename=keyfile)
+
+(stdin, stdout, stderr) = client.exec_command("gerrit version")
+for line in stdout.readlines():
+    print(line)
+
+# Avoid confusing "TypeError: 'NoneType' object is not callable" exception
+# https://github.com/paramiko/paramiko/issues/1078
+if client is not None:
+    client.close()
+    del client, stdin, stdout, stderr
--- a/web/conf/nginx.conf
+++ b/web/conf/nginx.conf
@ -0,0 +1,27 @@
+# /etc/nginx/conf.d/default.conf
+
+server {
+	listen 80 default_server;
+	listen [::]:80 default_server;
+
+	error_log /dev/stdout info;
+
+    root /var/www/localhost;
+
+
+    location ~ ^/data/(.*) {
+        alias  /data/www/$1;
+    }
+
+    location ~ ^/elastic-recheck\/data/(.*\.json) {
+        alias  /data/www/$1;
+    }
+
+    location / {
+        error_page 404 = @fallback;
+    }
+
+    location @fallback {
+        proxy_pass http://status.openstack.org;
+    }
+}
--- a/web/share/favicon.ico
+++ b/web/share/favicon.ico
--- a/web/share/gate.html
+++ b/web/share/gate.html
@ -1,58 +1,58 @@
-<html xmlns="http://www.w3.org/1999/xhtml"
-      xmlns:py="http://genshi.edgewall.org/"
+<html xmlns="https://www.w3.org/1999/xhtml"
+      xmlns:py="https://genshi.edgewall.org/"
      lang="en">
  <HEAD>
    <TITLE>Elastic Recheck</TITLE>

 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.min.js"></script>
+	src="/jquery.min.js"></script>
 <script type="text/javascript" src="handlebars-v2.0.0.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery-visibility.min.js"></script>
+	src="/jquery-visibility.min.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery-graphite.js"></script>
+	src="/jquery-graphite.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/common.js"></script>
+	src="/common.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.canvaswrapper.js"></script>
+	src="/jquery.canvaswrapper.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.js"></script>
+	src="/jquery.flot.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.saturated.js"></script>
+	src="/jquery.flot.saturated.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.uiConstants.js"></script>
+	src="/jquery.flot.uiConstants.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.browser.js"></script>
+	src="/jquery.flot.browser.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.colorhelpers.js"></script>
+	src="/jquery.colorhelpers.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.drawSeries.js"></script>
+	src="/jquery.flot.drawSeries.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.time.js"></script>
+	src="/jquery.flot.time.js"></script>
 <script type="text/javascript"
-  src="elastic-recheck.js"></script>
+  src="/elastic-recheck.js"></script>
 <script type='text/javascript'>
  var data_url = '/elastic-recheck/data/gate.json';
 </script>

    <!-- Google Fonts -->
-    <link href='http://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>
+    <link href='https://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>

    <!-- Framework CSS -->
-    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
-    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>
+    <link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
+    <link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>

    <!-- IE CSS -->
-    <!--[if lt IE 8]><link rel="stylesheet" href="http://www.openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->
+    <!--[if lt IE 8]><link rel="stylesheet" href="https://www.openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->

    <!-- OpenStack Specific CSS -->

-    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>
+    <link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>

    <!-- Page Specific CSS -->
-    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>
+    <link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>

-    <link rel="stylesheet" type="text/css" href="http://www.openstack.org/themes/openstack/css/main.css" />
+    <link rel="stylesheet" type="text/css" href="https://www.openstack.org/themes/openstack/css/main.css" />


    <!-- Project specific css -->
@ -88,7 +88,7 @@
           {{/if}}
           <div class="graph"></div>
           <a class="extlink" href="{{bug.logstash_url}}">Logstash</a>
-           <a class="extlink" href="http://bugs.launchpad.net/bugs/{{bug.number}}">Launchpad</a>
+           <a class="extlink" href="https://bugs.launchpad.net/bugs/{{bug.number}}">Launchpad</a>
      </div>
    </script>

--- a/web/share/index.html
+++ b/web/share/index.html
@ -1,59 +1,59 @@
-<html xmlns="http://www.w3.org/1999/xhtml"
-      xmlns:py="http://genshi.edgewall.org/"
+<html xmlns="https://www.w3.org/1999/xhtml"
+      xmlns:py="https://genshi.edgewall.org/"
      lang="en">
  <HEAD>
    <TITLE>Elastic Recheck</TITLE>

 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.min.js"></script>
+	src="/jquery.min.js"></script>
 <script type="text/javascript" src="handlebars-v2.0.0.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery-visibility.min.js"></script>
+	src="/jquery-visibility.min.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery-graphite.js"></script>
+	src="/jquery-graphite.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/common.js"></script>
+	src="/common.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.canvaswrapper.js"></script>
+	src="/jquery.canvaswrapper.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.js"></script>
+	src="/jquery.flot.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.saturated.js"></script>
+	src="/jquery.flot.saturated.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.uiConstants.js"></script>
+	src="/jquery.flot.uiConstants.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.browser.js"></script>
+	src="/jquery.flot.browser.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.colorhelpers.js"></script>
+	src="/jquery.colorhelpers.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.drawSeries.js"></script>
+	src="/jquery.flot.drawSeries.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.time.js"></script>
+	src="/jquery.flot.time.js"></script>
 <script type="text/javascript"
-  src="elastic-recheck.js"></script>
+  src="/elastic-recheck.js"></script>
 <script type='text/javascript'>
  var data_url = '/elastic-recheck/data/all.json';
 </script>

    <!-- Google Fonts -->
-    <link href='http://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>
+    <link href='https://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>

    <!-- Framework CSS -->
-    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
-    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>
+    <link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
+    <link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>

    <!-- IE CSS -->
-    <!--[if lt IE 8]><link rel="stylesheet" href="http://www.openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->
+    <!--[if lt IE 8]><link rel="stylesheet" href="https://www.openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->

    <!-- OpenStack Specific CSS -->

-    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>
+    <link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>

    <!-- Page Specific CSS -->
-    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>
+    <link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>

    <link rel="stylesheet" type="text/css"
-    href="http://www.openstack.org/themes/openstack/css/main.css" />
+    href="https://www.openstack.org/themes/openstack/css/main.css" />



@ -86,7 +86,7 @@
           {{/if}}
           <div class="graph"></div>
           <a class="extlink" href="{{bug.logstash_url}}">Logstash</a>
-           <a class="extlink" href="http://bugs.launchpad.net/bugs/{{bug.number}}">Launchpad</a>
+           <a class="extlink" href="https://bugs.launchpad.net/bugs/{{bug.number}}">Launchpad</a>
      </div>
    </script>
    <div class="container">
@ -96,7 +96,7 @@
            <li><a href="data/integrated_gate.html">Uncategorized Integrated Gate Jobs</a></li>
            <li><a href="data/others.html">Uncategorized</a></li>
        </ul>
-    <p>The elastic-recheck project uses Elasticsearch to classify and track OpenStack gate failures. Documentation can be found here: <a href="http://docs.openstack.org/infra/elastic-recheck/">http://docs.openstack.org/infra/elastic-recheck/</a>. You can also learn more by reading this post on the Elasticsearch blog: <a href="http://www.elasticsearch.org/blog/openstack-elastic-recheck-powered-elk-stack/">OpenStack elastic-recheck: powered by the elk stack</a>.</p>
+    <p>The elastic-recheck project uses Elasticsearch to classify and track OpenStack gate failures. Documentation can be found here: <a href="https://docs.openstack.org/infra/elastic-recheck/">docs.openstack.org/infra/elastic-recheck/</a>. You can also learn more by reading this post on the Elasticsearch blog: <a href="https://www.elasticsearch.org/blog/openstack-elastic-recheck-powered-elk-stack/">OpenStack elastic-recheck: powered by the elk stack</a>.</p>
    </div>

    <div id="vital-stats" class="container">
--- a/web/share/templates/base.html
+++ b/web/share/templates/base.html
@ -1,42 +1,42 @@
-<html xmlns="http://www.w3.org/1999/xhtml"
-      xmlns:py="http://genshi.edgewall.org/"
+<html xmlns="https://www.w3.org/1999/xhtml"
+      xmlns:py="https://genshi.edgewall.org/"
      lang="en">
  <HEAD>
    <TITLE>Elastic Recheck</TITLE>

 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.min.js"></script>
+	src="/jquery.min.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery-visibility.min.js"></script>
+	src="/jquery-visibility.min.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery-graphite.js"></script>
+	src="/jquery-graphite.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/common.js"></script>
+	src="/common.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.min.js"></script>
+	src="/jquery.flot.js"></script>
 <script type="text/javascript"
-	src="http://status.openstack.org/jquery.flot.time.min.js"></script>
+	src="/jquery.flot.time.js"></script>
 <script type="text/javascript"
-	src="elastic-recheck.js"></script>
+	src="/elastic-recheck.js"></script>

    <!-- Google Fonts -->
-    <link href='http://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>
+    <link href='https://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>

    <!-- Framework CSS -->
-    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
-    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>
+    <link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
+    <link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>

    <!-- IE CSS -->
    <!--[if lt IE 8]><link rel="stylesheet" href="http://www.openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->

    <!-- OpenStack Specific CSS -->

-    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>
+    <link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>

    <!-- Page Specific CSS -->
-    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>
+    <link rel="stylesheet" href="https://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>

-    <link rel="stylesheet" type="text/css" href="http://www.openstack.org/themes/openstack/css/main.css" />
+    <link rel="stylesheet" type="text/css" href="https://www.openstack.org/themes/openstack/css/main.css" />

    <!-- Project specific css -->
    <link rel="stylesheet" type="text/css" href="../styles/elastic-recheck.css" />