role: Add support and integration tests for postgresql

This adds support for specifying the "django.db.backends.postgresql"
django database engine, installing the required dependencies and setting
up the necessary configuration.

We're also adding a new package extras [postgresql] which will take care
of installing psycopg2 automatically.

This is self tested by integration tests which creates a postgresql
Docker container and runs the ara_api role configured to run against the
postgresql container. The tests are designed to run either locally or on Zuul.

Change-Id: I9bb7166d01c96d8b39361a55b1c9952de8fa9798

.zuul.d/zuul.yaml

@@ -64,6 +64,7 @@
     vars:
       ara_tests_cleanup: false
       ara_api_source: "{{ ansible_user_dir }}/src/opendev.org/recordsansible/ara"
+      ara_api_version: HEAD
       ansible_python_interpreter: /usr/bin/python3
 
 - job:
@@ -76,6 +77,17 @@
     parent: ansible-role-ara-api-base
     nodeset: fedora-latest
 
+# TODO: the install-docker role assumes running on Ubuntu
+- job:
+    name: ansible-role-ara-api-ubuntu-postgresql
+    parent: ansible-role-ara-api-base
+    nodeset: ubuntu-bionic
+    roles:
+      - zuul: zuul/zuul-jobs
+    pre-run: tests/install_docker.yaml
+    run: tests/with_postgresql.yaml
+    post-run: tests/ara_tests_post.yaml
+
 - job:
     name: ansible-role-ara-web-base
     parent: base
@@ -187,6 +199,7 @@
         - ansible-role-ara-tests-ubuntu-2.6
         - ansible-role-ara-api-ubuntu
         - ansible-role-ara-api-fedora
+        - ansible-role-ara-api-ubuntu-postgresql
         - ansible-role-ara-web-ubuntu
         - ansible-role-ara-web-fedora
         - ara-tox-linters
@@ -197,6 +210,7 @@
         - ansible-role-ara-tests-ubuntu-2.6
         - ansible-role-ara-api-ubuntu
         - ansible-role-ara-api-fedora
+        - ansible-role-ara-api-ubuntu-postgresql
         - ansible-role-ara-web-ubuntu
         - ansible-role-ara-web-fedora
         - ara-tox-linters

roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml

@@ -0,0 +1,49 @@
+---
+#  Copyright (c) 2019 Red Hat, Inc.
+#
+#  This file is part of ARA Records Ansible.
+#
+#  ARA Records Ansible is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  ARA Records Ansible is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with ARA Records Ansible. If not, see <http://www.gnu.org/licenses/>.
+
+- name: Retrieve list of missing required packages for postgresql
+  set_fact:
+    _postgresql_missing_packages: "{{ ara_api_postgresql_packages | difference(ansible_facts.packages.keys()) }}"
+
+# Only attempt to elevate privileges if there are any missing packages
+- when: _postgresql_missing_packages | length > 0
+  block:
+    - name: Install required packages
+      become: yes
+      package:
+        name: "{{ ara_api_postgresql_packages }}"
+        state: present
+  rescue:
+    - name: Fail due to missing packages
+      fail:
+        msg: "Failed to elevate privileges and install missing required packages. Install the following packages before running this role again: {{ _postgresql_missing_packages | join(' ') }}"
+
+# Django requires psycopg2 when using postgresql
+# https://docs.djangoproject.com/en/2.2/ref/databases/#postgresql-notes
+- name: Ensure psycopg2 is installed
+  pip:
+    name: psycopg2
+    state: present
+    virtualenv: "{{ ara_api_venv | bool | ternary(ara_api_venv_path, omit) }}"
+    virtualenv_python: python3
+
+- name: Run SQL migrations
+  environment:
+    ARA_SETTINGS: "{{ ara_api_settings }}"
+    PATH: "{{ path_with_virtualenv | default(omit) }}"
+  command: ara-manage migrate

roles/ara_api/vars/Fedora.yaml

@@ -21,3 +21,8 @@ ara_api_required_packages:
   - python3-virtualenv
   - python3-libselinux
   - policycoreutils-python-utils
+
+ara_api_postgresql_packages:
+  - postgresql
+  - postgresql-devel
+  - gcc

roles/ara_api/vars/Ubuntu.yaml

@@ -19,3 +19,8 @@
 ara_api_required_packages:
   - git
   - python3-venv
+
+ara_api_postgresql_packages:
+  - postgresql
+  - postgresql-server-dev-10
+  - gcc

setup.cfg

@@ -39,6 +39,8 @@ server=
     django-filter
     dynaconf[yaml]
     whitenoise
+postgresql=
+    psycopg2
 
 [build_sphinx]
 source-dir = doc/source

tests/install_docker.yaml

@@ -0,0 +1,29 @@
+---
+#  Copyright (c) 2019 Red Hat, Inc.
+#
+#  This file is part of ARA Records Ansible.
+#
+#  ARA Records Ansible is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  ARA Records Ansible is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with ARA Records Ansible. If not, see <http://www.gnu.org/licenses/>.
+
+# Runs the install-docker role exposed by Zuul:
+#  - https://opendev.org/zuul/zuul-jobs/src/branch/master/roles/install-docker
+#  - https://zuul-ci.org/docs/zuul-jobs/container-roles.html#role-install-docker
+
+# Some tests (such as with postgresql) requires Docker to set up a test database
+# server. You do not need to run this role if you have Docker installed already.
+
+- name: Install docker
+  hosts: all
+  roles:
+    - install-docker

tests/test_tasks.yaml

@@ -0,0 +1,90 @@
+#  Copyright (c) 2019 Red Hat, Inc.
+#
+#  This file is part of ARA Records Ansible.
+#
+#  ARA is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  ARA is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with ARA.  If not, see <http://www.gnu.org/licenses/>.
+
+# This is a list of tasks instead of a playbook. They are meant to be included
+# as post_tasks inside the same play context as our roles so they have access
+# to the play and role variables.
+
+- name: Set defaults for Ansible package name and version
+  set_fact:
+    ara_tests_ansible_name: "{{ ara_tests_ansible_name | default('ansible') }}"
+    ara_tests_ansible_version: "{{ ara_tests_ansible_version | default('latest') }}"
+
+# If a version is not explicitly set we want to make sure to
+# completely omit the version argument to pip, as it will be coming
+# from the long-form integration_ansible_name variable.  Additionally, if
+# the version is the special value "latest", then we also want to omit
+# any version number, but also set the package state to "latest".
+- name: Set Ansible version for installation
+  set_fact:
+    _install_ansible_version: "{{ ara_tests_ansible_version }}"
+  when: ara_tests_ansible_version not in ("", "latest")
+
+- name: Set Ansible package state for installation
+  set_fact:
+    _install_ansible_state: latest
+  when: ara_tests_ansible_version == "latest"
+
+- name: Install Ansible alongside ARA
+  pip:
+    name: "{{ ara_tests_ansible_name }}"
+    version: "{{ _install_ansible_version | default(omit, True) }}"
+    state: "{{ _install_ansible_state | default(omit, True) }}"
+    virtualenv: "{{ ara_api_venv | bool | ternary(ara_api_venv_path, omit) }}"
+    virtualenv_python: python3
+
+- name: Get ARA plugins directory
+  environment:
+    PATH: "{{ path_with_virtualenv | default(omit) }}"
+  command: python -m ara.setup.plugins
+  changed_when: false
+  register: ara_setup_plugins
+
+# These aren't in the same task (i.e, with loop) so we can tell individual test
+# runs apart easily rather than keeping all the output bundled in a single task.
+- environment:
+    ANSIBLE_CALLBACK_PLUGINS: "{{ ara_setup_plugins.stdout }}/callback"
+    ANSIBLE_ACTION_PLUGINS: "{{ ara_setup_plugins.stdout }}/action"
+    ARA_SETTINGS: "{{ ara_api_settings }}"
+  vars:
+    _ansible_playbook: "{{ ara_api_venv_path }}/bin/ansible-playbook -vvv"
+    _test_root: "{{ ara_api_source_checkout }}/tests/integration"
+  block:
+    # smoke.yaml tests setting ara_playbook_name in one of three plays
+    - name: Run smoke.yaml integration test
+      command: "{{ _ansible_playbook }} {{ _test_root }}/smoke.yaml"
+
+    - name: Run hosts.yaml integration test
+      command: "{{ _ansible_playbook }} {{ _test_root }}/hosts.yaml"
+
+    - name: Run import.yaml integration test
+      command: "{{ _ansible_playbook }} {{ _test_root }}/import.yaml"
+
+    # Tests setting ara_playbook_name as an extra var
+    - name: Run failed.yaml integration test
+      command: >
+        {{ _ansible_playbook }} {{ _test_root }}/failed.yaml -e ara_playbook_name="Failed playbook"
+      ignore_errors: yes
+
+    - name: Run incomplete.yaml integration test
+      shell: |
+        {{ _ansible_playbook }} {{ _test_root }}/incomplete.yaml &
+        sleep 5
+        kill $!
+      args:
+        executable: /bin/bash
+      ignore_errors: yes

tests/with_postgresql.yaml

@@ -0,0 +1,105 @@
+#  Copyright (c) 2019 Red Hat, Inc.
+#
+#  This file is part of ARA Records Ansible.
+#
+#  ARA is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  ARA is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with ARA.  If not, see <http://www.gnu.org/licenses/>.
+
+- name: Test the ARA API with postgresql
+  hosts: all
+  gather_facts: yes
+  vars:
+    ara_tests_cleanup: true
+    ara_api_root_dir: /tmp/ara-integration-tests/.ara
+    ara_api_secret_key: testing
+    ara_api_debug: true
+    ara_api_log_level: DEBUG
+    ara_api_database_engine: django.db.backends.postgresql
+    ara_api_database_name: ara
+    ara_api_database_user: ara
+    ara_api_database_password: password
+    ara_api_database_host: 127.0.0.1
+    ara_api_database_port: 5432
+  pre_tasks:
+    # TODO: This fails if the docker python library isn't installed but we can
+    # recover by running the command manually. The Ansible provided by Zuul
+    # does not have the module installed.
+    - name: Start a postgresql container
+      docker_container:
+        name: ara_tests_postgresql
+        image: postgres:10
+        state: started
+        ports:
+          - "{{ ara_api_database_port }}:{{ ara_api_database_port }}"
+        env:
+          POSTGRES_DB: "{{ ara_api_database_name }}"
+          POSTGRES_USER: "{{ ara_api_database_user }}"
+          POSTGRES_PASSWORD: "{{ ara_api_database_password }}"
+      ignore_errors: yes
+      register: _docker_run
+
+    - name: Start a postgresql container without docker python library
+      command: |
+        docker run -d \
+          --name ara_tests_postgresql \
+          -p {{ ara_api_database_port }}:{{ ara_api_database_port }} \
+          -e POSTGRES_DB={{ ara_api_database_name }} \
+          -e POSTGRES_USER={{ ara_api_database_user }} \
+          -e POSTGRES_PASSWORD={{ ara_api_database_password }} \
+          postgres:10
+      when:
+        - _docker_run is failed
+        - "'Failed to import docker or docker-py' in _docker_run.msg"
+  tasks:
+    - block:
+        - name: Set up the API with the ara_api role
+          import_role:
+            name: ara_api
+
+        # These are tasks rather than a standalone playbook to give us an easy
+        # access to all the variables within the same play.
+        - include_tasks: test_tasks.yaml
+
+        - name: Set up postgre password file for passwordless commands
+          lineinfile:
+            path: "{{ ansible_user_dir }}/.pgpass"
+            create: yes
+            line: "{{ ara_api_database_host }}:{{ ara_api_database_port }}:{{ ara_api_database_name }}:{{ ara_api_database_user }}:{{ ara_api_database_password }}"
+            mode: 0600
+
+        # Dump is suffixed by .txt so we don't need magic mimetypes when
+        # viewing on the web.
+        - name: Dump database file
+          command: |
+            pg_dump -w \
+              --host={{ ara_api_database_host }} \
+              --username={{ ara_api_database_user }} \
+              --dbname={{ ara_api_database_name }} \
+              --file={{ ara_api_base_dir }}/pg_dump.sql.txt
+
+      always:
+        - when: ara_tests_cleanup | bool
+          block:
+            - name: Clean up the postgresql container
+              docker_container:
+                name: ara_tests_postgresql
+                state: absent
+              ignore_errors: yes
+              register: _docker_rm
+
+            - name: Remove the postgresql container without the docker python library
+              command: docker rm -f ara_tests_postgresql
+              ignore_errors: yes
+              when:
+                - _docker_rm is failed
+                - "'Failed to import docker or docker-py' in _docker_rm.msg"