diff --git a/.zuul.d/jobs.yaml b/.zuul.d/jobs.yaml index 78d670f6..307d8469 100644 --- a/.zuul.d/jobs.yaml +++ b/.zuul.d/jobs.yaml @@ -26,6 +26,18 @@ override-checkout: stable-2.9 pre-run: tests/zuul_pre_multinode_networking.yaml +- job: + name: ara-api-postgresql + parent: ara-api-database-backends + nodeset: ara-database-server-multinode + description: | + Deploys the ARA API server on Ubuntu 18.04 as well as Fedora 30 and + tests it against a central PostgreSQL server installed on CentOS 8. + The job exercises the ara_api Ansible role, the ARA Ansible plugins, the + ARA API clients as well as the API itself. + run: tests/with_postgresql.yaml + post-run: tests/zuul_post_with_postgresql.yaml + - job: name: ara-api-mysql parent: ara-api-database-backends diff --git a/.zuul.d/project.yaml b/.zuul.d/project.yaml index 659087a3..44cf6a78 100644 --- a/.zuul.d/project.yaml +++ b/.zuul.d/project.yaml @@ -6,11 +6,11 @@ check: jobs: - ara-api-mysql + - ara-api-postgresql - ansible-role-ara-api-fedora-devel - ansible-role-ara-api-fedora-2.9 - ansible-role-ara-api-ubuntu-2.8 - ansible-role-ara-api-ubuntu-2.7 - - ansible-role-ara-api-ubuntu-postgresql - ansible-role-ara-api-fedora-distributed-sqlite - ansible-role-ara-api-fedora-packages: voting: false @@ -22,11 +22,11 @@ gate: jobs: - ara-api-mysql + - ara-api-postgresql - ansible-role-ara-api-fedora-devel - ansible-role-ara-api-fedora-2.9 - ansible-role-ara-api-ubuntu-2.8 - ansible-role-ara-api-ubuntu-2.7 - - ansible-role-ara-api-ubuntu-postgresql - ansible-role-ara-api-fedora-distributed-sqlite - ansible-role-ara-web-ubuntu - ansible-role-ara-web-fedora diff --git a/.zuul.d/role-jobs.yaml b/.zuul.d/role-jobs.yaml index 5ba7b679..ac2eeceb 100644 --- a/.zuul.d/role-jobs.yaml +++ b/.zuul.d/role-jobs.yaml @@ -75,14 +75,6 @@ override-checkout: stable-2.7 # Database server jobs -- job: - name: ansible-role-ara-api-ubuntu-postgresql - parent: ansible-role-ara-api-ubuntu - roles: - - zuul: zuul/zuul-jobs - pre-run: tests/install_docker.yaml - run: tests/with_postgresql.yaml - - job: name: ansible-role-ara-api-fedora-distributed-sqlite parent: ansible-role-ara-api-fedora diff --git a/tests/install_docker.yaml b/tests/install_docker.yaml deleted file mode 100644 index 565880be..00000000 --- a/tests/install_docker.yaml +++ /dev/null @@ -1,29 +0,0 @@ ---- -# Copyright (c) 2019 Red Hat, Inc. -# -# This file is part of ARA Records Ansible. -# -# ARA Records Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ARA Records Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with ARA Records Ansible. If not, see . - -# Runs the install-docker role exposed by Zuul: -# - https://opendev.org/zuul/zuul-jobs/src/branch/master/roles/install-docker -# - https://zuul-ci.org/docs/zuul-jobs/container-roles.html#role-install-docker - -# Some tests (such as with postgresql) requires Docker to set up a test database -# server. You do not need to run this role if you have Docker installed already. - -- name: Install docker - hosts: all - roles: - - install-docker diff --git a/tests/vars/postgresql_tests.yaml b/tests/vars/postgresql_tests.yaml new file mode 100644 index 00000000..d1983f1a --- /dev/null +++ b/tests/vars/postgresql_tests.yaml @@ -0,0 +1,16 @@ +ara_tests_cleanup: true +ara_api_root_dir: "{{ ansible_user_dir }}/.ara-tests" +ara_api_secret_key: testing +ara_api_debug: true +ara_api_log_level: DEBUG +# Set to 0 because tests could be using the offline client +ara_api_database_conn_max_age: 0 +ara_api_database_engine: django.db.backends.postgresql +ara_api_database_name: ara +ara_api_database_user: ara +ara_api_database_password: password +ara_api_database_port: 5432 +# The host is defined dynamically based on the address of the database server +# ara_api_database_host: 127.0.0.1 +_postgresql_container_name: ara-postgresql-tests +_postgresql_image_name: docker.io/postgres:10 diff --git a/tests/with_postgresql.yaml b/tests/with_postgresql.yaml index 3d9dc240..f5e01a5e 100644 --- a/tests/with_postgresql.yaml +++ b/tests/with_postgresql.yaml @@ -15,92 +15,75 @@ # You should have received a copy of the GNU General Public License # along with ARA. If not, see . -- name: Test the ARA API with postgresql - hosts: all +- name: Start a postgresql container + hosts: ara-database-server gather_facts: yes - vars: - ara_tests_cleanup: true - ara_api_root_dir: "{{ ansible_user_dir }}/.ara-tests" - ara_api_secret_key: testing - ara_api_debug: true - ara_api_log_level: DEBUG - ara_api_database_engine: django.db.backends.postgresql - ara_api_database_name: ara - ara_api_database_user: ara - ara_api_database_password: password - ara_api_database_host: 127.0.0.1 - ara_api_database_port: 5432 - pre_tasks: - # TODO: This fails if the docker python library isn't installed but we can - # recover by running the command manually. The Ansible provided by Zuul - # does not have the module installed. - - name: Start a postgresql container - docker_container: - name: ara_tests_postgresql - image: postgres:10 - state: started - ports: - - "{{ ara_api_database_port }}:{{ ara_api_database_port }}" - env: - POSTGRES_DB: "{{ ara_api_database_name }}" - POSTGRES_USER: "{{ ara_api_database_user }}" - POSTGRES_PASSWORD: "{{ ara_api_database_password }}" - ignore_errors: yes - register: _docker_run + vars_files: + - vars/postgresql_tests.yaml + tasks: + - name: Install podman + become: yes + package: + name: podman + state: present - - name: Start a postgresql container without docker python library + - name: Start a postgresql container command: | - docker run -d \ - --name ara_tests_postgresql \ + podman run -d \ + --name {{ _postgresql_container_name }} \ -p {{ ara_api_database_port }}:{{ ara_api_database_port }} \ -e POSTGRES_DB={{ ara_api_database_name }} \ -e POSTGRES_USER={{ ara_api_database_user }} \ -e POSTGRES_PASSWORD={{ ara_api_database_password }} \ - postgres:10 - when: - - _docker_run is failed - - "'Failed to import docker or docker-py' in _docker_run.msg or 'Failed to import the required Python library' in _docker_run.msg" + {{ _postgresql_image_name }} + + # podman doesn't appear to be able to listen on ipv6 yet: https://github.com/containers/libpod/issues/3245 + # If we have a node on IPv6, redirect the traffic from v6 to v4 with socat + - when: ansible_host | ipv6 != false + become: yes + block: + - name: Install socat + package: + name: socat + state: present + + - name: Setup systemd service + copy: + dest: /etc/systemd/system/socat-postgresql.service + owner: root + group: root + mode: 0644 + content: | + [Unit] + Description=socat postgresql ipv6 to ipv4 + [Service] + ExecStart=/usr/bin/socat TCP6-LISTEN:5432,fork,bind=[{{ ansible_host }}] TCP4:127.0.0.1:5432 + + [Install] + WantedBy=multi-user.target + + - name: Start socat network redirection for postgresql over ipv6 + service: + name: socat-postgresql + state: started + enabled: yes + daemon_reload: yes + +- name: Deploy and test ARA API with postgresql + hosts: ara-api-server + gather_facts: yes + vars_files: + - vars/postgresql_tests.yaml tasks: - - block: - - name: Set up the API with the ara_api role - include_role: - name: ara_api - public: yes + - name: Set database server host + set_fact: + ara_api_database_host: "{{ hostvars['database-server']['ansible_host'] }}" - # These are tasks rather than a standalone playbook to give us an easy - # access to all the variables within the same play. - - include_tasks: test_tasks.yaml + - name: Set up the API with the ara_api Ansible role + include_role: + name: ara_api + public: yes - - name: Set up postgre password file for passwordless commands - lineinfile: - path: "{{ ansible_user_dir }}/.pgpass" - create: yes - line: "{{ ara_api_database_host }}:{{ ara_api_database_port }}:{{ ara_api_database_name }}:{{ ara_api_database_user }}:{{ ara_api_database_password }}" - mode: 0600 - - # Dump is suffixed by .txt so we don't need magic mimetypes when - # viewing on the web. - - name: Dump database file - command: | - pg_dump -w \ - --host={{ ara_api_database_host }} \ - --username={{ ara_api_database_user }} \ - --dbname={{ ara_api_database_name }} \ - --file={{ ara_api_base_dir }}/pg_dump.sql.txt - - always: - - when: ara_tests_cleanup | bool - block: - - name: Clean up the postgresql container - docker_container: - name: ara_tests_postgresql - state: absent - ignore_errors: yes - register: _docker_rm - - - name: Remove the postgresql container without the docker python library - command: docker rm -f ara_tests_postgresql - ignore_errors: yes - when: - - _docker_rm is failed - - "'Failed to import docker or docker-py' in _docker_rm.msg" + # These are tasks rather than a standalone playbook to give us an easy + # access to all the variables within the same play. + - include_tasks: test_tasks.yaml diff --git a/tests/zuul_post_with_postgresql.yaml b/tests/zuul_post_with_postgresql.yaml new file mode 100644 index 00000000..c5dacc01 --- /dev/null +++ b/tests/zuul_post_with_postgresql.yaml @@ -0,0 +1,54 @@ +--- +# Copyright (c) 2020 Red Hat, Inc. +# +# This file is part of ARA Records Ansible. +# +# ARA Records Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ARA Records Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ARA Records Ansible. If not, see . + +- name: Recover postgresql server data + hosts: ara-database-server + gather_facts: yes + vars_files: + - vars/postgresql_tests.yaml + tasks: + - name: Ensure pg_dump is installed + become: yes + package: + name: postgresql + state: present + + # The zuul-output directory is retrieved by the fetch-output role + # https://opendev.org/zuul/zuul-jobs/src/branch/master/roles/fetch-output + - name: Create log directory + file: + path: "{{ ansible_user_dir }}/zuul-output/logs" + state: directory + recurse: yes + + - name: Set up postgre password file for passwordless commands + lineinfile: + path: "{{ ansible_user_dir }}/.pgpass" + create: yes + line: "127.0.0.1:{{ ara_api_database_port }}:{{ ara_api_database_name }}:{{ ara_api_database_user }}:{{ ara_api_database_password }}" + mode: 0600 + + # Dump is suffixed by .txt so we don't need magic mimetypes when + # viewing on the web. + - name: Dump database file + command: | + pg_dump -w \ + --host=127.0.0.1 \ + --username={{ ara_api_database_user }} \ + --dbname={{ ara_api_database_name }} \ + --file={{ ansible_user_dir }}/zuul-output/logs/pg_dump.sql.txt