From 1217799b1bef82dcd2a6c69eb795c398cbac0d1b Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 10 Jul 2019 15:33:31 -0500 Subject: [PATCH] Add molecule testing This change adds molecule testing using a simple base job and pre|run playbooks. The test will be executed via a native zuul job and will ensure we're exercising all of the available code path's as provide by this role. Two molecule scenarios will be executed whenever any change is made to this role * default - runs through the typical main code path * login - tests a secure docker registry ensuring our login capabilities are never broken. Documentation in the readme has been added to show how local testing can be run. A bindep.txt file has been added to ensure zuul knows how to install our required base packages. Closes-Bug: #1835657 Related-Bug: #1833584 Change-Id: I48f74b69c5d29dce4a576fa96e79563a4b484469 Signed-off-by: Kevin Carter --- .yamllint | 11 +++++ README.rst | 58 ++++++++++++++++++++++++++ bindep.txt | 33 +++++++++++++++ molecule/default/molecule.yml | 54 +++++++++++++++++++++++++ molecule/default/playbook.yml | 24 +++++++++++ molecule/default/prepare.yml | 34 ++++++++++++++++ molecule/login/molecule.yml | 54 +++++++++++++++++++++++++ molecule/login/playbook.yml | 30 ++++++++++++++ molecule/login/prepare.yml | 57 ++++++++++++++++++++++++++ tasks/docker-distribution.yml | 2 + tasks/docker-login.yml | 2 + tasks/docker-update.yml | 2 + tasks/docker.yml | 2 + tasks/main.yml | 2 + zuul.d/base.yaml | 15 +++++++ zuul.d/layout.yaml | 2 + zuul.d/playbooks/pre.yml | 74 ++++++++++++++++++++++++++++++++++ zuul.d/playbooks/run-local.yml | 14 +++++++ zuul.d/playbooks/run.yml | 13 ++++++ 19 files changed, 483 insertions(+) create mode 100644 .yamllint create mode 100644 bindep.txt create mode 100644 molecule/default/molecule.yml create mode 100644 molecule/default/playbook.yml create mode 100644 molecule/default/prepare.yml create mode 100644 molecule/login/molecule.yml create mode 100644 molecule/login/playbook.yml create mode 100644 molecule/login/prepare.yml create mode 100644 zuul.d/base.yaml create mode 100644 zuul.d/playbooks/pre.yml create mode 100644 zuul.d/playbooks/run-local.yml create mode 100644 zuul.d/playbooks/run.yml diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..ad0be76 --- /dev/null +++ b/.yamllint @@ -0,0 +1,11 @@ +extends: default + +rules: + braces: + max-spaces-inside: 1 + level: error + brackets: + max-spaces-inside: 1 + level: error + line-length: disable + truthy: disable diff --git a/README.rst b/README.rst index f9c3f93..a462a6d 100644 --- a/README.rst +++ b/README.rst @@ -92,3 +92,61 @@ License ------- Apache 2.0 + + +Running local testing +--------------------- + +Local testing of this role can be done in a number of ways. + +Mimic Zuul +~~~~~~~~~~ + +Sometimes its nessisary to setup a test that will mimic what the OpenStack gate +will do (Zuul). To run tests that minic the gate, `python-virtualenv` `git`, +`gcc`, and `ansible` are required. + +.. code-block:: shell + + $ sudo yum install python-virtualenv git gcc + + +Once the packages are installed, create a python virtual environment. + +.. code-block:: shell + + $ python -m virtualenv --system-site-packages ~/test-python + $ ~/test-python/bin/pip install pip setuptools --upgrade + + +Now install the latest Ansible + +.. code-block:: shell + + $ ~/test-python/bin/pip install ansible + + +With Ansible installed, activate the virtual environment and run the +`run-local.yml` test playbook. + +.. code-block:: shell + + $ source ~/test-python/bin/activate + (test-python) $ ansible-playbook -i 'localhost,' \ + -e "tripleo_src=$(realpath --relative-to="${HOME}" "$(pwd)")" \ + -e "ansible_user=${USER}" \ + -e "ansible_user_dir=${HOME}" \ + -e "ansible_connection=local" \ + zuul.d/playbooks/run-local.yml + + +Running Molecule directly +~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is also possible to test this role using molecule directly. When running +tests directly it is assumed all of the dependencies are setup and ready to +run on the local workstation. When + +.. code-block:: shell + + $ molecule test --all diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..cd16325 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,33 @@ +# This file facilitates OpenStack-CI package installation +# before the execution of any tests. +# +# See the following for details: +# - https://docs.openstack.org/infra/bindep/ +# - https://opendev.org/opendev/bindep/ +# +# Even if the role does not make use of this facility, it +# is better to have this file empty, otherwise OpenStack-CI +# will fall back to installing its default packages which +# will potentially be detrimental to the tests executed. + +# The gcc compiler +gcc + +# Base requirements for RPM distros +gcc-c++ [platform:rpm] +git [platform:rpm] +libffi-devel [platform:rpm] +openssl-devel [platform:rpm] +python-devel [platform:rpm] +python2-dnf [platform:fedora] +python-virtualenv [platform:rpm] + +# For SELinux +libselinux-python [platform:rpm] +libsemanage-python [platform:redhat] + +# Required for compressing collected log files in CI +gzip + +# Required to build language docs +gettext diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml new file mode 100644 index 0000000..7409d6b --- /dev/null +++ b/molecule/default/molecule.yml @@ -0,0 +1,54 @@ +--- +driver: + name: delegated + options: + managed: false + login_cmd_template: >- + ssh + -o UserKnownHostsFile=/dev/null + -o StrictHostKeyChecking=no + -o Compression=no + -o TCPKeepAlive=yes + -o VerifyHostKeyDNS=no + -o ForwardX11=no + -o ForwardAgent=no + {instance} + ansible_connection_options: + ansible_connection: ssh + +log: true + +platforms: + - name: instance + +provisioner: + name: ansible + config_options: + defaults: + fact_caching: jsonfile + fact_caching_connection: /tmp/molecule/facts + inventory: + hosts: + all: + hosts: + instance: + ansible_host: localhost + log: true + env: + ANSIBLE_STDOUT_CALLBACK: yaml + ANSIBLE_ROLES_PATH: "${ANSIBLE_ROLES_PATH}:${HOME}/zuul-jobs/roles" + +scenario: + name: default + test_sequence: + - prepare + - converge + - verify + +lint: + enabled: false + +verifier: + name: testinfra + lint: + name: flake8 diff --git a/molecule/default/playbook.yml b/molecule/default/playbook.yml new file mode 100644 index 0000000..9d89234 --- /dev/null +++ b/molecule/default/playbook.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +- name: Converge + become: true + hosts: all + roles: + - role: ansible-role-container-registry + container_registry_deploy_docker: true + container_registry_deploy_docker_distribution: true diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml new file mode 100644 index 0000000..c335449 --- /dev/null +++ b/molecule/default/prepare.yml @@ -0,0 +1,34 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + + +- name: Prepare + hosts: all + become: true + gather_facts: true + pre_tasks: + - name: set basic user fact + set_fact: + ansible_user: "{{ lookup('env', 'USER') }}" + when: + - ansible_user is undefined + + - name: set basic home fact + set_fact: + ansible_user_dir: "{{ lookup('env', 'HOME') }}" + when: + - ansible_user_dir is undefined diff --git a/molecule/login/molecule.yml b/molecule/login/molecule.yml new file mode 100644 index 0000000..19c52b5 --- /dev/null +++ b/molecule/login/molecule.yml @@ -0,0 +1,54 @@ +--- +driver: + name: delegated + options: + managed: false + login_cmd_template: >- + ssh + -o UserKnownHostsFile=/dev/null + -o StrictHostKeyChecking=no + -o Compression=no + -o TCPKeepAlive=yes + -o VerifyHostKeyDNS=no + -o ForwardX11=no + -o ForwardAgent=no + {instance} + ansible_connection_options: + ansible_connection: ssh + +log: true + +platforms: + - name: instance + +provisioner: + name: ansible + config_options: + defaults: + fact_caching: jsonfile + fact_caching_connection: /tmp/molecule/facts + inventory: + hosts: + all: + hosts: + instance: + ansible_host: localhost + log: true + env: + ANSIBLE_STDOUT_CALLBACK: yaml + ANSIBLE_ROLES_PATH: "${ANSIBLE_ROLES_PATH}:${HOME}/zuul-jobs/roles" + +scenario: + name: login + test_sequence: + - prepare + - converge + - verify + +lint: + enabled: false + +verifier: + name: testinfra + lint: + name: flake8 diff --git a/molecule/login/playbook.yml b/molecule/login/playbook.yml new file mode 100644 index 0000000..c29731d --- /dev/null +++ b/molecule/login/playbook.yml @@ -0,0 +1,30 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +- name: Converge + become: false + hosts: all + vars: + container_registry_logins: + localhost:5000: + testuser: testpassword + tasks: + - include_role: + name: ansible-role-container-registry + tasks_from: docker-login + vars: + ansible_python_interpreter: "{{ ansible_user_dir }}/test-python/bin/python" diff --git a/molecule/login/prepare.yml b/molecule/login/prepare.yml new file mode 100644 index 0000000..d925463 --- /dev/null +++ b/molecule/login/prepare.yml @@ -0,0 +1,57 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + + +- name: Prepare + hosts: all + become: true + gather_facts: true + pre_tasks: + - name: set basic user fact + set_fact: + ansible_user: "{{ lookup('env', 'USER') }}" + when: + - ansible_user is undefined + + - name: set basic home fact + set_fact: + ansible_user_dir: "{{ lookup('env', 'HOME') }}" + when: + - ansible_user_dir is undefined + + - include_role: + name: ansible-role-container-registry + tasks_from: docker + + tasks: + - name: Install docker-sdk + pip: + name: docker + virtualenv: "{{ ansible_user_dir }}/test-python" + virtualenv_site_packages: true + + - name: Create a docker registry + become: true + shell: |- + docker pull ubuntu:16.04 + docker tag ubuntu:16.04 localhost:5000/my-ubuntu + mkdir auth + docker run --entrypoint htpasswd registry:2 -Bbn testuser testpassword > auth/htpasswd + docker container stop registry + docker run -d -p 5000:5000 --restart=always --name registry -v "$(pwd)"/auth:/auth -e "REGISTRY_AUTH=htpasswd" -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd registry:2 + args: + creates: /root/auth/htpasswd diff --git a/tasks/docker-distribution.yml b/tasks/docker-distribution.yml index 35b6138..f2f9c4e 100644 --- a/tasks/docker-distribution.yml +++ b/tasks/docker-distribution.yml @@ -1,3 +1,5 @@ +--- + # tasks file for ansible-role-container-registry - name: ensure docker-distribution is installed diff --git a/tasks/docker-login.yml b/tasks/docker-login.yml index 21207be..c649657 100644 --- a/tasks/docker-login.yml +++ b/tasks/docker-login.yml @@ -1,3 +1,5 @@ +--- + # tasks file for ansible-role-container-registry - name: perform docker login diff --git a/tasks/docker-update.yml b/tasks/docker-update.yml index 8c45e87..3a20677 100644 --- a/tasks/docker-update.yml +++ b/tasks/docker-update.yml @@ -1,3 +1,5 @@ +--- + # tasks file for ansible-role-container-registry # the tasks will ensure docker is up to date. diff --git a/tasks/docker.yml b/tasks/docker.yml index a0ef995..7638f77 100644 --- a/tasks/docker.yml +++ b/tasks/docker.yml @@ -1,3 +1,5 @@ +--- + # tasks file for ansible-role-container-registry # NOTE(mfedosin): In order to verify that we have already configured docker diff --git a/tasks/main.yml b/tasks/main.yml index 9b28e13..6ecd454 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,3 +1,5 @@ +--- + # tasks file for ansible-role-container-registry - include_tasks: docker.yml diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml new file mode 100644 index 0000000..4351480 --- /dev/null +++ b/zuul.d/base.yaml @@ -0,0 +1,15 @@ +- job: + description: Base ansible-role-container-registry job + name: ansible-role-container-registry-centos-7-base + nodeset: centos-7 + parent: base + pre-run: + - zuul.d/playbooks/pre.yml + timeout: 1800 + voting: true +- job: + description: Run molecule testing + name: ansible-role-container-registry-centos-7-molecule + parent: ansible-role-container-registry-centos-7-base + run: + - zuul.d/playbooks/run.yml diff --git a/zuul.d/layout.yaml b/zuul.d/layout.yaml index 83a318e..0a405ec 100644 --- a/zuul.d/layout.yaml +++ b/zuul.d/layout.yaml @@ -1,10 +1,12 @@ - project: check: jobs: + - ansible-role-container-registry-centos-7-molecule - openstack-tox-linters - tripleo-ci-centos-7-undercloud-containers gate: jobs: + - ansible-role-container-registry-centos-7-molecule - openstack-tox-linters - tripleo-ci-centos-7-undercloud-containers post: diff --git a/zuul.d/playbooks/pre.yml b/zuul.d/playbooks/pre.yml new file mode 100644 index 0000000..24a0ca1 --- /dev/null +++ b/zuul.d/playbooks/pre.yml @@ -0,0 +1,74 @@ +--- + +- hosts: all + vars: + molecule_requirements: + - ansible + - ansi2html + - docker + - pytest + - pytest-cov + - pytest-html + - pytest-xdist + - mock + - molecule>=2.22rc1 + + tasks: + - name: set basic user fact + fail: + msg: >- + The variable `ansible_user` set this option and try again. On the + CLI this can be defined with "-e ansible_user=${USER}" + when: + - ansible_user is undefined + + - name: set basic home fact + fail: + msg: >- + The variable `ansible_user_dir` set this option and try again. On + the CLI this can be defined with "-e ansible_user_dir=${HOME}" + when: + - ansible_user_dir is undefined + + - name: Ensure the user has a .ssh directory + file: + path: "{{ ansible_user_dir }}/.ssh" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: "0700" + + - name: Create ssh key pair + user: + name: "{{ ansible_user }}" + generate_ssh_key: true + ssh_key_bits: 2048 + ssh_key_file: "{{ ansible_user_dir }}/.ssh/id_rsa" + + - name: Slurp pub key + slurp: + src: "{{ ansible_user_dir ~ '/.ssh/id_rsa.pub' }}" + register: pub_key + + - name: Ensure can ssh to can connect to localhost + authorized_key: + user: "{{ ansible_user }}" + key: "{{ pub_key['content'] | b64decode }}" + + - name: Ensure output dirs + file: + path: "{{ ansible_user_dir }}/zuul-output/logs" + state: directory + + - name: Get the zuul/zuul-jobs repo + git: + repo: https://opendev.org/zuul/zuul-jobs + dest: "{{ ansible_user_dir }}/zuul-jobs" + version: master + force: true + + - name: Setup test-python + pip: + name: "{{ molecule_requirements }}" + virtualenv: "{{ ansible_user_dir }}/test-python" + virtualenv_site_packages: true diff --git a/zuul.d/playbooks/run-local.yml b/zuul.d/playbooks/run-local.yml new file mode 100644 index 0000000..65d4d5e --- /dev/null +++ b/zuul.d/playbooks/run-local.yml @@ -0,0 +1,14 @@ +--- + +- hosts: all + tasks: + - name: set basic zuul fact + set_fact: + zuul: + project: + src_dir: "{{ tripleo_src }}" + ansible_connection: ssh + +- import_playbook: pre.yml + +- import_playbook: run.yml diff --git a/zuul.d/playbooks/run.yml b/zuul.d/playbooks/run.yml new file mode 100644 index 0000000..8d7a514 --- /dev/null +++ b/zuul.d/playbooks/run.yml @@ -0,0 +1,13 @@ +--- + +- hosts: all + environment: + ANSIBLE_LOG_PATH: "{{ ansible_user_dir }}/zuul-output/logs/ansible-execution.log" + tasks: + - name: Run role test job + shell: |- + . {{ ansible_user_dir }}/test-python/bin/activate + molecule test --all + args: + chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}" + executable: /bin/bash