diff --git a/.dockerignore b/.dockerignore index e6dbcd91e4..230524f721 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,5 @@ -rally-jobs tests contrib test-requirements.txt tox.ini +.tox \ No newline at end of file diff --git a/.zuul.d/zuul.yaml b/.zuul.d/zuul.yaml index f6c8c661fb..11765e9081 100644 --- a/.zuul.d/zuul.yaml +++ b/.zuul.d/zuul.yaml @@ -1,3 +1,29 @@ +- secret: + name: dockerhub_credentials + data: + username: !encrypted/pkcs1-oaep + - k7GCjmH8P2DqliVc5ihbr+yYwVeylzShrmoP3qIMb1jx4Mkgn8YDY54vf84HT5xEDAuJw + y/ndAtvVY/4WF0MsRZ/p4ZaAZx1FyV90ZgNXKNAgizMmvBIZoVEbFN5gimJn8DVnvcm5R + vIezNzsgWQoNiOOuxwB3j6Fq+40K6mBg2k1ceuvqX/DAKqXgNdX/V5SPxBYek4WkofxUs + gDFdHt3aMIty+Gd/fW4D/ufGAurW4CUbFmiAVyZagXzq8NKXCKUOGFeNvL/tftXhT2bd1 + PczUz8Nhur1Z7nYeqc5K1TnN35OlNP0IT4KG8vst33nX4ycSkFOpbkdvflnchlysEvn/+ + Z1NMFKLxQvi/k4rMa2jBv7O6xs9y1LOZMSHwG0Y2U8lV6m1FqrDyi9h22fcBGEtOyp0KV + V9DmSFRCNb7lhd7v7vaztzL8yqrGlezrJYELodUvYLy2AwZi+FnkUVFw7dB4D5m/fjMCf + ej/LRO4XiUpVpLUL0BOxLNXVHqqrbROwPkp2ekJL3WxMwhAGSxlAuMwvPU1KZzRhrdnG7 + Ysm12YV5qgd6b6XZ8X7vELxAiqZPCjSN+66FPZW+Q6I5UVX+IxIgndgxgkoHJOgg9Wjwb + kKDgYM8EE5Ks4/Rv9jfNXxXvH/cQFqWIMHHbsZfmV6QcLfJxYXDRZ98r6KTzwA= + password: !encrypted/pkcs1-oaep + - JxlQJhxRBxgyqfHyUPVbS4pagJxkA6sdJyPf5p84iWFa1nWbIDTbpzhGYhYBqMt1YrXFf + NWGHzs7rJ/hp4MXSRHWVG4lEi5O6RFx/KEWiru+rGJwr7CyIfyyVORlRGlnABrc/LYyAe + LoSsQ5GM98Q1wERbhiZq7zsKSf0xIazFDmf34+bIx6Q9r4nlWdCQuQJ9N07UT4aRsl3WS + hOlUp8ekxBiQRtS9lRUPLUmJ8OzHTzXjJ6V0fM3MhDGA3YnH0Tj9r1D7cmwhsTy5ZT36k + bt2WCknxJ9tiJND5/NMvzo9T82apgoqeH8HCWDjE6PEdmJdYLiiW4aCk22D2M6a0+qvQI + fD8ZeEbrlJLvf58ldGgCJfl1Rpql0FspgnERBkCL4xSlyMQa8+S5R2S/SN6hAsbLRm/jZ + zUt3C4n8pJJvBDUBGEK+lqersyJtx5jEiz6xh0FVDVzFxBn/fUbNvQQH+zJAe4V1j1hHw + MBu4V2p20Qpjv2+wF2yzwT+tnrCkaeyuBfBLGRVdn4nCz630X1eaOwqh0hu1ZPDiwNBs4 + hNv/EV7oaaTlk4VftYRwYocdK/pvj1Mi0QCea+hC01vuoqJE8EK05oOD1KThOTrbwXQX5 + CDzBLc/Vyd4tFnAXPwEJDEKFNo25R7nmDBYTQIuiHSOSZHJis+r5UwvHfbuJPk= + - job: name: rally-install-base parent: base @@ -23,12 +49,25 @@ - job: name: rally-docker-build - parent: base + parent: build-docker-image nodeset: ubuntu-bionic - pre-run: tests/ci/playbooks/rally-docker-build-pre.yaml - run: tests/ci/playbooks/rally-docker-build-run.yaml + run: tests/ci/playbooks/build-and-check-docker-image.yaml + post-run: tests/ci/playbooks/fetch-html-and-json-reports.yaml timeout: 1800 + +- job: + name: rally-docker-build-and-push + parent: build-docker-image + nodeset: ubuntu-bionic + run: tests/ci/playbooks/build-check-and-push-docker-image.yaml + post-run: tests/ci/playbooks/fetch-html-and-json-reports.yaml + timeout: 1800 + secrets: + name: docker_credentials + secret: dockerhub_credentials + pass-to-parent: false + - job: name: rally-database-migration parent: base @@ -71,4 +110,4 @@ - rally-install-ubuntu-bionic - rally-install-centos-7 - rally-install-centos-8 - - rally-docker-build + - rally-docker-build-and-push diff --git a/etc/docker/README.md b/DOCKER_README.md similarity index 100% rename from etc/docker/README.md rename to DOCKER_README.md diff --git a/etc/docker/Dockerfile b/Dockerfile similarity index 90% rename from etc/docker/Dockerfile rename to Dockerfile index 24d0883c5c..3119dc8eca 100644 --- a/etc/docker/Dockerfile +++ b/Dockerfile @@ -9,8 +9,7 @@ RUN apt-get update && apt-get install --yes sudo python3-dev python3-pip vim git echo "rally ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/00-rally-user && \ mkdir /rally && chown -R rally:rally /rally -COPY ./src /rally/source -COPY ./motd /etc/motd +COPY ./ /rally/source WORKDIR /rally/source RUN pip3 install . --constraint upper-constraints.txt && \ @@ -19,13 +18,15 @@ RUN pip3 install . --constraint upper-constraints.txt && \ mkdir -p /etc/rally && \ echo "[database]" > /etc/rally/rally.conf && \ echo "connection=sqlite:////home/rally/.rally/rally.db" >> /etc/rally/rally.conf -RUN echo '[ ! -z "$TERM" -a -r /etc/motd ] && cat /etc/motd' >> /etc/bash.bashrc # Cleanup pip RUN rm -rf /root/.cache/ +COPY ./etc/motd_for_docker /etc/motd +RUN echo '[ ! -z "$TERM" -a -r /etc/motd ] && cat /etc/motd' >> /etc/bash.bashrc + USER rally ENV HOME /home/rally -RUN mkdir -p /home/rally/.rally && rally db recreate +RUN mkdir -p /home/rally/.rally && rally db recreate # Docker volumes have specific behavior that allows this construction to work. # Data generated during the image creation is copied to volume only when it's diff --git a/etc/docker/README_FIRST.rst b/etc/docker/README_FIRST.rst deleted file mode 100644 index bf94f70aa0..0000000000 --- a/etc/docker/README_FIRST.rst +++ /dev/null @@ -1,11 +0,0 @@ -ReadMe of /etc/docker dir -===================================== - -We are using automated docker image builds on `Docker Hub -`_ which allows to reduce time of making new releases. - -Docker Hub has one specific feature - each time it builds new image, it -updates the description of the image. The description it takes from README file -of the same directory as Dockerfile is located. That is why Dockerfile is -placed to the separate directory with 2 README files: one for Docker Hub, -another one (this one) for explanation of situation. diff --git a/etc/docker/hooks/post_checkout b/etc/docker/hooks/post_checkout deleted file mode 100644 index ad9331b92f..0000000000 --- a/etc/docker/hooks/post_checkout +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# NOTE(andreykurilin): We have automated builds for new docker images at -# Docker Cloud. To publish a custom, docker specific README file, we need -# to apply a workaround: change the context directory for building images -# from the root repo dir '/' to '/etc/docker/' where we store our Dockerfile. -# In this case, Docker Cloud will use local README, i.e /etc/docker/README.md -# To make source code available from Dockerfile we need to apply -# post_checkout docker cloud hook (see [*] for more details). Our custom hook -# copies rally framework code under the directory which is available from -# docker build context, i.e from /etc/docker -# -# [*] - https://docs.docker.com/docker-cloud/builds/advanced/#custom-build-phase-hooks - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -mkdir ${DIR}/../src/ -rsync -av --progress ${DIR}/../../../ ${DIR}/../src/ --exclude etc/docker diff --git a/etc/docker/motd b/etc/motd_for_docker similarity index 100% rename from etc/docker/motd rename to etc/motd_for_docker diff --git a/tests/ci/playbooks/build-and-check-docker-image.yaml b/tests/ci/playbooks/build-and-check-docker-image.yaml new file mode 100644 index 0000000000..25d13df3cb --- /dev/null +++ b/tests/ci/playbooks/build-and-check-docker-image.yaml @@ -0,0 +1,4 @@ +- hosts: all + name: Execute the similar to `tox -e self` job + roles: + - build_docker_image diff --git a/tests/ci/playbooks/build-check-and-push-docker-image.yaml b/tests/ci/playbooks/build-check-and-push-docker-image.yaml new file mode 100644 index 0000000000..6629232811 --- /dev/null +++ b/tests/ci/playbooks/build-check-and-push-docker-image.yaml @@ -0,0 +1,5 @@ +- hosts: all + name: Execute the similar to `tox -e self` job + roles: + - build_docker_image + - push_docker_image diff --git a/tests/ci/playbooks/rally-docker-build-pre.yaml b/tests/ci/playbooks/rally-docker-build-pre.yaml deleted file mode 100644 index dd892bb140..0000000000 --- a/tests/ci/playbooks/rally-docker-build-pre.yaml +++ /dev/null @@ -1,18 +0,0 @@ -- hosts: all - name: Execute post checkout hook as like hub.docker.com does - tasks: - - name: Install docker - shell: - cmd: | - sudo apt install --yes apt-transport-https ca-certificates curl software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" - sudo apt update --yes - apt-cache policy docker-ce - sudo apt install --yes docker-ce - sudo systemctl status docker - executable: /bin/bash - - - name: Execute post_checkout hook - shell: - cmd: "sudo /bin/bash {{ zuul.project.src_dir }}/etc/docker/hooks/post_checkout" diff --git a/tests/ci/playbooks/rally-docker-build-run.yaml b/tests/ci/playbooks/rally-docker-build-run.yaml deleted file mode 100644 index d3ebbf6dd7..0000000000 --- a/tests/ci/playbooks/rally-docker-build-run.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- hosts: all - name: a run script for rally-install-* jobs - tasks: - - name: Build docker image for Rally - shell: - cmd: "sudo docker build -t rally:dev ./" - chdir: '{{ zuul.project.src_dir }}/etc/docker' diff --git a/tests/ci/playbooks/rally-install-run.yaml b/tests/ci/playbooks/rally-install-run.yaml index ffda4985ec..322edbc8c4 100644 --- a/tests/ci/playbooks/rally-install-run.yaml +++ b/tests/ci/playbooks/rally-install-run.yaml @@ -19,8 +19,18 @@ chdir: '{{ zuul.project.src_dir }}' cmd: "sudo pip3 install --constraint ./upper-constraints.txt ./" + - name: Create direcotry for html and json reports + shell: + executable: /bin/bash + chdir: '{{ zuul.project.src_dir }}' + cmd: mkdir .test_results + - name: Execute the similar script as `tox -e self` shell: - chdir: '{{ zuul.project.src_dir }}' executable: /bin/bash - cmd: './tests/ci/rally_self_job.sh ./rally-jobs/self-rally.yaml' + chdir: '{{ zuul.project.src_dir }}' + cmd: > + python3 ./tests/ci/rally_self_job.py + --task ./rally-jobs/self-rally.yaml + --plugins-path ./rally-jobs/plugins + --results-dir ./.test_results diff --git a/tests/ci/playbooks/roles/build_docker_image/defaults/main.yaml b/tests/ci/playbooks/roles/build_docker_image/defaults/main.yaml new file mode 100644 index 0000000000..2cfb9b82b5 --- /dev/null +++ b/tests/ci/playbooks/roles/build_docker_image/defaults/main.yaml @@ -0,0 +1,3 @@ +docker_repository: xrally/xrally +docker_image_version: latest +docker_image_tag: "{{ docker_repository }}:{{ docker_image_version }}" \ No newline at end of file diff --git a/tests/ci/playbooks/roles/build_docker_image/tasks/main.yaml b/tests/ci/playbooks/roles/build_docker_image/tasks/main.yaml new file mode 100644 index 0000000000..49ebdf60dd --- /dev/null +++ b/tests/ci/playbooks/roles/build_docker_image/tasks/main.yaml @@ -0,0 +1,24 @@ +- name: Build xrally/xrally docker image + shell: + chdir: '{{ zuul.project.src_dir }}' + cmd: 'docker build -t {{ docker_image_tag }} ./' + +- name: List available docker images + shell: docker image list + +- name: Prepare directories to mount to the docker container for persistency + shell: + chdir: '{{ zuul.project.src_dir }}' + cmd: | + mkdir -p .test_results + sudo chown 65500 .test_results + +- name: Execute the similar wokrloads as `tox -e self` + shell: + cmd: > + python3 {{ zuul.project.src_dir }}/tests/ci/rally_self_job.py + --task /rally/source/rally-jobs/self-rally.yaml + --plugins-path /rally/source/rally-jobs/plugins + --rally-cmd 'docker run -v '$(realpath {{ zuul.project.src_dir }})'/.test_results:/home/rally/.rally {{ docker_image_tag }}' + --results-dir /home/rally/.rally + --without-tmp-sqlite diff --git a/tests/ci/playbooks/roles/push_docker_image/defaults/main.yaml b/tests/ci/playbooks/roles/push_docker_image/defaults/main.yaml new file mode 100644 index 0000000000..32dff599eb --- /dev/null +++ b/tests/ci/playbooks/roles/push_docker_image/defaults/main.yaml @@ -0,0 +1,4 @@ +docker_repository: xrally/xrally +docker_image_version: latest +docker_image_tag: "{{ docker_repository }}:{{ docker_image_version }}" +docker_file: "{{ zuul.project.src_dir }}/DOCKER_README.md" diff --git a/tests/ci/playbooks/roles/push_docker_image/tasks/main.yaml b/tests/ci/playbooks/roles/push_docker_image/tasks/main.yaml new file mode 100644 index 0000000000..631da6070e --- /dev/null +++ b/tests/ci/playbooks/roles/push_docker_image/tasks/main.yaml @@ -0,0 +1,42 @@ +- name: Log in to dockerhub + command: "docker login -u '{{ docker_credentials.username }}' -p '{{ docker_credentials.password }}'" + no_log: true + +- name: Upload image to dockerhub + command: "docker push {{ docker_image_tag }}" + register: result + until: result.rc == 0 + retries: 3 + delay: 30 + +- name: Read README file for docker + command: "cat {{ docker_file }}" + register: docker_readme + +- name: Get dockerhub JWT token + no_log: true + uri: + url: "https://hub.docker.com/v2/users/login/" + body_format: json + body: + username: '{{ docker_credentials.username }}' + password: '{{ docker_credentials.password }}' + register: jwt_token + delay: 5 + retries: 3 + until: jwt_token and jwt_token.status==200 + +- name: Update README at Docker HUB + no_log: true + uri: + url: "https://hub.docker.com/v2/repositories/{{ docker_repository }}/" + method: "PATCH" + body_format: json + headers: + Authorization: "JWT {{ jwt_token.json.token }}" + body: + full_description: "{{ docker_readme.stdout }}" + register: repository_metadata + delay: 5 + retries: 3 + until: repository_metadata and repository_metadata.status==200 diff --git a/tests/ci/rally_self_job.py b/tests/ci/rally_self_job.py new file mode 100644 index 0000000000..23a8e4211a --- /dev/null +++ b/tests/ci/rally_self_job.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import subprocess +import sys +import uuid + + +def create_cfg_for_using_sqlite(): + """Create a config file for rally with path to temporary sqlite database""" + run_id = str(uuid.uuid4())[:8] + + cfg_path = f"/tmp/self_rally_{run_id}.conf" + + cfg = (f"[database]\n" + f"connection = sqlite:////tmp/self-rally-{run_id}.sqlite") + with open(cfg_path, "w") as f: + f.write(cfg) + + return "--config-file", cfg_path + + +def _print_with_wrapper(*messages): + print("#" * 80) + for m in messages: + print(f"# {m}") + print("#" * 80) + + +class Rally(object): + def __init__(self, rally_cmd): + self._rally_cmd = rally_cmd + + def __call__(self, cmd, debug=False, description=None): + if not isinstance(cmd, (tuple, list)): + cmd = cmd.split(" ") + + final_cmd = self._rally_cmd[:] + if debug: + final_cmd.append("--debug") + final_cmd.extend(cmd) + + messages = [f"Calling: {' '.join(final_cmd)}"] + if description: + messages.append(f"Description: {description}") + _print_with_wrapper(*messages) + + p = subprocess.Popen( + final_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + for line in iter(p.stdout.readline, b''): + print(line.rstrip().decode("utf-8")) + p.wait() + if p.returncode != 0: + _print_with_wrapper( + f"Command returned non-zero exit status {p.returncode}.") + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser( + "rally_self_job", + description="A script that ensures that Rally works after installation" + ) + parser.add_argument( + "--results-dir", type=str, default=".test_results", + help="A path to directory to place rally results execution." + ) + parser.add_argument( + "--task", type=str, required=True, + help="A simple task to launch." + ) + parser.add_argument( + "--without-tmp-sqlite", action="store_false", + dest="prepare_sqlite", + help="Do not create a tmp sqlite db for Rally" + ) + parser.add_argument( + "--rally-cmd", type=str, default="rally", + help="An override 'rally' entry point for all commands." + ) + parser.add_argument( + "--plugins-path", type=str, required=True, + help="A path to external rally plugins that include " + "'FakePlugin.testplugin'" + ) + + args = parser.parse_args() + + rally_cmd = args.rally_cmd.split(" ") + if "--config-file" in rally_cmd: + raise Exception( + "It is restricted to use '--config-file' at rally cmd.") + + if args.prepare_sqlite: + rally_cmd.extend(create_cfg_for_using_sqlite()) + + rally_cmd.append(f"--plugin-paths={args.plugins_path}") + + rally = Rally(rally_cmd) + + rally("--version") + rally("plugin show --name FakePlugin.testplugin", debug=True, + description="Ensure plugins loading") + + rally("db create", description="Initialize temporary sqlite database") + + rally("env create --name=self", + description="Create empty environment") + rally(f"task start {args.task}") + + results_dir = args.results_dir + rally(f"task report --html-static --out {results_dir}/self_report.html") + rally(f"task report --json --out {results_dir}/self_report.json") + rally(f"task sla-check") + + +if __name__ == "__main__": + main() diff --git a/tests/ci/rally_self_job.sh b/tests/ci/rally_self_job.sh deleted file mode 100755 index ac3aa44bfd..0000000000 --- a/tests/ci/rally_self_job.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -ex -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -TASK_FILE=$1 -PLUGIN_PATHS=rally-jobs/plugins - -mkdir -p .test_results -HTML_REPORT=.test_results/self_report.html -JSON_REPORT=.test_results/self_report.json - -RND=$(head /dev/urandom | tr -dc a-z0-9 | head -c 5) -TMP_RALLY_CONF="/tmp/self-rally-$RND.conf" -TMP_RALLY_DB="/tmp/self-rally-$RND.sqlite" -DBCONNSTRING="sqlite:///$TMP_RALLY_DB" -RALLY="rally --config-file $TMP_RALLY_CONF" - -# Create temp cfg -cp etc/rally/rally.conf.sample $TMP_RALLY_CONF -sed -i.bak "s|#connection =.*|connection = \"$DBCONNSTRING\"|" $TMP_RALLY_CONF - -# ensure plugins loading -$RALLY --debug --plugin-paths=$PLUGIN_PATHS plugin show --name FakePlugin.testplugin - -# Create db -$RALLY db create - -# Create self deployment -$RALLY -d env create --name=self - -# Run task -set +e -$RALLY -d --plugin-paths=$PLUGIN_PATHS task start $TASK_FILE -if [ $? -eq 1 ]; then - exit 1 -fi -set -e - -$RALLY task report --html-static --out $HTML_REPORT -$RALLY task report --json --out $JSON_REPORT - -# Check sla (this may fail the job) -$RALLY task sla-check diff --git a/tests/unit/doc/test_docker_readme.py b/tests/unit/doc/test_docker_readme.py index 9965df0f41..73f2a6f9c6 100644 --- a/tests/unit/doc/test_docker_readme.py +++ b/tests/unit/doc/test_docker_readme.py @@ -23,7 +23,6 @@ from tests.unit import test ROOT_DIR = os.path.dirname(os.path.dirname(rally.__file__)) -DOCKER_DIR = os.path.join(ROOT_DIR, "etc", "docker") class DockerReadmeTestCase(test.TestCase): @@ -51,7 +50,7 @@ class DockerReadmeTestCase(test.TestCase): return releases def test_mentioned_latest_version(self): - full_path = os.path.join(ROOT_DIR, "etc/docker/README.md") + full_path = os.path.join(ROOT_DIR, "DOCKER_README.md") with open(full_path) as f: readme = f.read() diff --git a/tox.ini b/tox.ini index 8a830cb59f..8e69eaeddb 100644 --- a/tox.ini +++ b/tox.ini @@ -96,10 +96,16 @@ commands = bindep [testenv:self-py38] basepython = python3.8 -commands = {toxinidir}/tests/ci/rally_self_job.sh {toxinidir}/rally-jobs/self-rally.yaml +commands = + find . -type f -name "*.pyc" -delete + mkdir -p .test_results + python3 {toxinidir}/tests/ci/rally_self_job.py --task {toxinidir}/rally-jobs/self-rally.yaml --plugins-path {toxinidir}/rally-jobs/plugins [testenv:self] -commands = {toxinidir}/tests/ci/rally_self_job.sh {toxinidir}/rally-jobs/self-rally.yaml +commands = \ + find . -type f -name "*.pyc" -delete + mkdir -p {toxinidir}/.test_results + python3 {toxinidir}/tests/ci/rally_self_job.py --task {toxinidir}/rally-jobs/self-rally.yaml --plugins-path {toxinidir}/rally-jobs/plugins [pytest] filterwarnings =