Updated/corrected service_status

This patch updates service_status check, importing what was done in
oooq-extras back in 2019[1].

This will allow to actually ensure only tripleo related services are
checked.

The legacy Docker support is still here in order to allow checking on
older releases.

This patch also adds some fancy molecule testing.

[1] https://review.opendev.org/#/c/637729/

Change-Id: I9016497305b4ec1a1e49b4b1c3cb23d219a42458

roles/service_status/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+service_status_podman_opt: ''

roles/service_status/molecule/default/molecule.yml

@@ -11,6 +11,7 @@ platforms:
     pkg_extras: python-setuptools python-enum34 python-netaddr ruby epel-release PyYAML
     easy_install:
       - pip
+    command: /sbin/init
     environment: &env
       http_proxy: "{{ lookup('env', 'http_proxy') }}"
       https_proxy: "{{ lookup('env', 'https_proxy') }}"
@@ -18,6 +19,7 @@ platforms:
   - name: fedora28
     hostname: fedora28
     image: fedora:28
+    command: /sbin/init
     pkg_extras: python*-setuptools python*-enum python*-netaddr ruby PyYAML
     environment:
       <<: *env

roles/service_status/molecule/default/playbook.yml

@@ -20,7 +20,6 @@
   gather_facts: false
 
   tasks:
-    - name: Warn developers about the lack of molecule testing
-      fail:
-        msg: >-
-          This role needs molecule tests!
+    - name: Full check with defaults
+      include_role:
+        name: service_status

roles/service_status/molecule/docker/Dockerfile.j2

@@ -0,0 +1,37 @@
+# Molecule managed
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]

roles/service_status/molecule/docker/molecule.yml

@@ -0,0 +1,61 @@
+---
+driver:
+  name: docker
+
+log: true
+
+platforms:
+  - name: centos7
+    hostname: centos7
+    image: centos:7
+    pkg_extras: python-setuptools python-enum34 python-netaddr ruby epel-release PyYAML
+    easy_install:
+      - pip
+    command: /sbin/init
+    capabilities:
+      - SYS_ADMIN
+    privileged: true
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    environment: &env
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+
+  - name: fedora28
+    hostname: fedora28
+    image: fedora:28
+    command: /sbin/init
+    capabilities:
+      - SYS_ADMIN
+    privileged: true
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    pkg_extras: python*-setuptools python*-enum python*-netaddr ruby PyYAML python*-libselinux
+    environment:
+      <<: *env
+
+provisioner:
+  name: ansible
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "../../../../library"
+
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - verify
+    - destroy
+
+lint:
+  enabled: false
+
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

roles/service_status/molecule/docker/playbook.yml

@@ -0,0 +1,59 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Converge
+  hosts: all
+  gather_facts: false
+  become: true
+
+  tasks:
+    - name: "Check containers - docker version, no service"
+      include_role:
+        name: service_status
+        tasks_from: containers.yaml
+
+    - name: "Check containers - docker version, with service"
+      block:
+        - name: Activate docker service
+          service:
+            name: docker
+            state: started
+            enabled: true
+
+        - name: Catch failure
+          block:
+            - name: Run check
+              include_role:
+                name: service_status
+                tasks_from: containers.yaml
+
+          rescue:
+            - name: Clear host errors
+              meta: clear_host_errors
+
+            - name: Test output
+              debug:
+                msg: |
+                  Success finding broken containers
+
+            - name: End play
+              meta: end_play
+
+    - name: Fail if this point is reached
+      fail:
+        msg: |
+          Did not find broken containers

roles/service_status/molecule/docker/prepare.yml

@@ -0,0 +1,65 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Prepare
+  hosts: all
+  gather_facts: false
+
+  tasks:
+    - name: install docker
+      package:
+        name: docker
+
+    - name: fake docker exe
+      copy:
+        dest: /usr/bin/docker
+        mode: 0755
+        content: |
+          #!/bin/sh
+          echo 'thirsty_goldwasser Exited (0) 12 seconds ago'
+          echo 'fedora28 Exited (255) 7 hours ago'
+          echo 'centos7 Exited (255) 7 hours ago'
+
+    - name: docker unit override basedir
+      file:
+        path: /etc/systemd/system/docker.service.d
+        state: directory
+
+    - name: fake docker unit
+      copy:
+        dest: /etc/systemd/system/docker.service.d/override.conf
+        content: |
+          [Unit]
+          After=network.target
+          Wants=
+          Requires=
+
+          [Service]
+          Type=simple
+          ExecStart=
+          ExecStart=/usr/bin/fake
+          Restart=
+
+    - name: fake docker exec for unit
+      copy:
+        dest: /usr/bin/fake
+        mode: 0755
+        content: |
+          #!/bin/sh
+          while true; do
+            sleep 5;
+          done

roles/service_status/molecule/podman/Dockerfile.j2

@@ -0,0 +1,37 @@
+# Molecule managed
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]

roles/service_status/molecule/podman/molecule.yml

@@ -0,0 +1,49 @@
+---
+driver:
+  name: docker
+
+log: true
+
+platforms:
+  - name: centos7
+    hostname: centos7
+    image: centos:7
+    pkg_extras: python-setuptools python-enum34 python-netaddr ruby epel-release PyYAML
+    easy_install:
+      - pip
+    command: /sbin/init
+    environment: &env
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+
+  - name: fedora28
+    hostname: fedora28
+    image: fedora:28
+    command: /sbin/init
+    pkg_extras: python*-setuptools python*-enum python*-netaddr ruby PyYAML
+    environment:
+      <<: *env
+
+provisioner:
+  name: ansible
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "../../../../library"
+
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - verify
+    - destroy
+
+lint:
+  enabled: false
+
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

roles/service_status/molecule/podman/playbook.yml

@@ -0,0 +1,46 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Converge
+  hosts: all
+  gather_facts: false
+  vars:
+    service_status_podman_opt: '--storage-driver=vfs'
+
+  tasks:
+    - name: Check podman container state
+      block:
+        - name: Detect failed podman containers
+          include_role:
+            name: service_status
+            tasks_from: containers.yaml
+      rescue:
+        - name: Clear host errors
+          meta: clear_host_errors
+
+        - name: Test output
+          debug:
+            msg: |
+              Properly detected failed container
+
+        - name: End play now
+          meta: end_play
+
+    - name: Fail if we get to this point
+      fail:
+        msg: |
+          Did not detect failed container

roles/service_status/molecule/podman/prepare.yml

@@ -0,0 +1,39 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Prepare
+  hosts: all
+  gather_facts: false
+
+  tasks:
+    - name: install podman
+      package:
+        name: podman
+
+    - name: Create libpod arbo
+      file:
+        path: '/var/lib/containers/{{ item }}'
+        state: directory
+      loop:
+        - storage
+        - storage/libpod
+
+    - name: Insert failed container DB
+      copy:
+        src: ./bolt_state.db
+        dest: /var/lib/containers/storage/libpod/bolt_state.db
+        setype: container_var_lib_t

roles/service_status/molecule/systemd/Dockerfile.j2

@@ -0,0 +1,37 @@
+# Molecule managed
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]

roles/service_status/molecule/systemd/molecule.yml

@@ -0,0 +1,49 @@
+---
+driver:
+  name: docker
+
+log: true
+
+platforms:
+  - name: centos7
+    hostname: centos7
+    image: centos:7
+    pkg_extras: python-setuptools python-enum34 python-netaddr ruby epel-release PyYAML
+    easy_install:
+      - pip
+    command: /sbin/init
+    environment: &env
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+
+  - name: fedora28
+    hostname: fedora28
+    image: fedora:28
+    command: /sbin/init
+    pkg_extras: python*-setuptools python*-enum python*-netaddr ruby PyYAML
+    environment:
+      <<: *env
+
+provisioner:
+  name: ansible
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "../../../../library"
+
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - verify
+    - destroy
+
+lint:
+  enabled: false
+
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

roles/service_status/molecule/systemd/playbook.yml

@@ -0,0 +1,44 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Converge
+  hosts: all
+  gather_facts: false
+
+  tasks:
+    - name: Check service
+      block:
+        - name: Run validation
+          include_role:
+            name: service_status
+            tasks_from: systemd.yaml
+      rescue:
+        - name: Clear errors
+          meta: clear_host_errors
+
+        - name: Test output
+          debug:
+            msg: |
+              Successfully detected failed unit
+
+        - name: End play
+          meta: end_play
+
+    - name: Fail if this point is reached
+      fail:
+        msg: |
+          Did not detect failed unit

roles/service_status/molecule/systemd/prepare.yml

@@ -0,0 +1,39 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Prepare
+  hosts: all
+  gather_facts: false
+
+  tasks:
+    - name: Create fake, failing unit
+      copy:
+        dest: /etc/systemd/system/tripleo_failed-unit.service
+        content: |
+          [Unit]
+          After=network.target
+
+          [Service]
+          Type=simple
+          ExecStart=/usr/bin/false
+
+    - name: Enable and start broken thing
+      ignore_errors: true
+      service:
+        name: tripleo_failed-unit
+        state: started
+        enabled: true

roles/service_status/tasks/containers.yaml

@@ -0,0 +1,59 @@
+---
+- name: Gather package facts
+  package_facts:
+    manager: auto
+
+- name: Is docker running
+  systemd:
+    name: docker
+  register: docker_svc
+  when: ansible_facts.packages['docker'] is defined
+
+- name: Do we have podman
+  stat:
+    path: /usr/bin/podman
+  register: podman_stat
+
+- name: Podman related block
+  when: podman_stat.stat.exists
+  block:
+    - name: Get failed containers for podman
+      become: true
+      shell: |
+        podman {{ service_status_podman_opt }} ps -a --filter 'status=exited' --format {{ "'{{ .Names }} {{ .Status }}'" }}
+      register: failed_podman
+
+    - name: Fail if we detect failed podman container
+      fail:
+        msg: |
+          Failed container detected.
+          On CI, please check the following locations
+          /var/log/extras/failed_containers.log
+          /var/log/extras/podman
+      when: item is not match(".* Exited \(0\) .* ago")
+      loop: "{{ failed_podman.stdout_lines }}"
+
+- name: Docker related block
+  when:
+    - ansible_facts.packages['docker'] is defined
+    - docker_svc.status['SubState'] == 'running'
+  block:
+    - name: Get failed containers from docker
+      become: true
+      shell: >
+        {% raw %}
+        docker ps -a --filter 'status=exited' --format '{{ .Names }} {{ .Status }}'
+        {% endraw %}
+      register: failed_docker
+
+    - name: Fail if we detect failed docker container
+      fail:
+        msg: |
+          Failed container detected.
+          On CI, please check the following locations
+          /var/log/extras/failed_containers.log
+          /var/log/extras/docker
+      when:
+        - failed_docker is defined
+        - item is not match(".* Exited \(0\) .* ago")
+      loop: "{{ failed_docker.stdout_lines }}"

roles/service_status/tasks/main.yaml

@@ -1,16 +1,3 @@
 ---
-- name: get failed systemd units
-  shell: >
-    systemctl list-units --failed --plain --no-legend --no-pager |
-    awk '{print $1}'
-  register: systemd_status
-  changed_when: false
-
-- name: Fails if we find failed units
-  assert:
-    fail_msg: >
-      Failed units were found: {{ systemd_status.stdout_lines|join(',') }}
-    success_msg: >
-      No failed unit were found
-    that:
-      - systemd_status.stdout_lines|length == 0
+- include_tasks: containers.yaml
+- include_tasks: systemd.yaml

roles/service_status/tasks/systemd.yaml

@@ -0,0 +1,13 @@
+---
+- name: Get failed services from Systemd
+  shell: >
+    systemctl list-units --failed --plain --no-legend --no-pager "tripleo_*"
+  register: systemd_state
+  changed_when: false
+
+- name: Fails if we find failed systemd units
+  assert:
+    that:
+      - systemd_state.stdout_lines|length == 0
+    fail_msg: "The following services failed {{ systemd_state.stdout_lines }}"
+    success_msg: "All tripleo units are working fine"