Migrate linting to pre-commit

Follows the same linting configuration that was implemented first in
tripleo-quickstart-extras which makes used of pre-commit tool for
managing all linters.

This also avoids problems where a new linter release may break our
gates because pre-commit always pins versions.

Removes ansible from requirements.txt as it needs to be listed only
in ansible-requirements.txt

Change-Id: Ia229d3d58763d743bd19ad9099d7907561f3c77f
Depends-On: https://review.openstack.org/#/c/626000/

.ansible-lint

@@ -0,0 +1,15 @@
+parseable: true
+rulesdir:
+    - ./ci-scripts/ansible_rules/
+quiet: false
+skip_list:
+    - ANSIBLE0006  # Using command rather than module we have a few use cases
+    # where we need to use curl and rsync
+    - ANSIBLE0007  # Using command rather than an argument to e.g file
+    # we have a lot of 'rm' command and we should use file module instead
+    - ANSIBLE0010  # Package installs should not use latest.
+    # Sometimes we need to update some packages.
+    - ANSIBLE0013  # Use Shell only when shell functionality is required
+    - ANSIBLE0016  # Tasks that run when changed should likely be handlers
+    # this requires refactoring roles, skipping for now
+verbosity: 1

.pre-commit-config.yaml

@@ -0,0 +1,41 @@
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: mixed-line-ending
+      - id: check-byte-order-marker
+      - id: check-executables-have-shebangs
+      - id: check-merge-conflict
+      - id: debug-statements
+      - id: flake8
+        additional_dependencies:
+          - hacking<1.2.0,>=1.1.0
+      - id: check-yaml
+        files: .*\.(yaml|yml)$
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: v1.13.0
+    hooks:
+      - id: yamllint
+        files: \.(yaml|yml)$
+        types: [file, yaml]
+        entry: yamllint --strict -f parsable
+  - repo: https://github.com/ansible/ansible-lint
+    rev: v4.0.0
+    hooks:
+      - id: ansible-lint
+        files: \.(yaml|yml)$
+        entry: ansible-lint --force-color -v
+  - repo: https://github.com/openstack-dev/bashate.git
+    rev: 0.6.0
+    hooks:
+      - id: bashate
+        entry: bashate --error . --verbose --ignore=E006,E040
+        # Run bashate check for all bash scripts
+        # Ignores the following rules:
+        # E006: Line longer than 79 columns (as many scripts use jinja
+        #       templating, this is very difficult)
+        # E040: Syntax error determined using `bash -n` (as many scripts
+        #       use jinja templating, this will often fail and the syntax
+        #       error will be discovered in execution anyway)

.yamllint

@@ -0,0 +1,6 @@
+---
+extends: default
+
+rules:
+  line-length:
+    max: 180

README.md

@@ -134,7 +134,7 @@ An example playbook is provided in tests/test.yml:
 
     - hosts: undercloud
       gather_facts: true
-      become: yes
+      become: true
       become_method: sudo
       become_user: stack
       roles:
@@ -153,7 +153,7 @@ scripts using this example playbook (duplicate from
     ---
     - hosts: undercloud
       gather_facts: true
-      become: yes
+      become: true
       become_method: sudo
       become_user: stack
       roles:

ci-scripts/ansible-lint.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-# ANSIBLE0006: Using command rather than module
-#   we have a few use cases where we need to use curl and rsync
-# ANSIBLE0007: Using command rather than an argument to e.g file
-#   we have a lot of 'rm' command and we should use file module instead
-# ANSIBLE0010: Package installs should not use latest.
-#   Sometimes we need to update some packages.
-# ANSIBLE0012: Commands should not change things if nothing needs doing
-# ANSIBLE0013: Use Shell only when shell functionality is required
-# ANSIBLE0016: Tasks that run when changed should likely be handlers
-#   this requires refactoring roles, skipping for now
-SKIPLIST="ANSIBLE0006,ANSIBLE0007,ANSIBLE0010,ANSIBLE0012,ANSIBLE0013,ANSIBLE0016"
-
-ansible-lint -vvv -x $SKIPLIST ./ -R -r ./ci-scripts/ansible_rules

infrared_plugin/main.yml

@@ -3,7 +3,7 @@
 
 - hosts: undercloud
   gather_facts: true
-  become: yes
+  become: true
   become_method: sudo
   become_user: stack
 

meta/main.yml

@@ -1,3 +1,4 @@
+---
 galaxy_info:
   author: your name
   description: your description

requirements.txt

@@ -1,2 +1 @@
 pbr>=1.6
-ansible

tasks/common/controller_post_scripts.yml

@@ -1,3 +1,4 @@
+---
 - name: register controller nodes
   shell: |
     source {{ undercloud_rc }}

tasks/common/convert_ceph_params.yaml

@@ -35,7 +35,7 @@
           copy:
             src: "{{ ceph_env }}"
             dest: "{{ working_dir}}/pre_upgrade_ceph_osd_hieradata_env.yaml"
-            remote_src: yes
+            remote_src: true
 
         - name: register environment file
           command: "cat {{ ceph_env }}"

tasks/common/convert_cli_opts_params.yaml

@@ -1,3 +1,4 @@
+---
 - name: register control-scale opts
   shell: |
     grep -oP "control-scale\ \d+"  {{ overcloud_deploy_script }}
@@ -93,9 +94,16 @@
   template:
     src: fast-forward-upgrade/cli_opts_params.yaml.j2
     dest: "{{working_dir}}/cli_opts_params.yaml"
-    force: no
+    force: false
   when:
-    - control_scale|succeeded or compute_scale|succeeded or ceph_scale|succeeded or control_flavor|succeeded or compute_flavor|succeeded or ceph_flavor|succeeded or ntp_server|succeeded
+    - >
+      control_scale|succeeded or
+      compute_scale|succeeded or
+      ceph_scale|succeeded or
+      control_flavor|succeeded or
+      compute_flavor|succeeded or
+      ceph_flavor|succeeded or
+      ntp_server|succeeded
 
 - name: check "{{working_dir}}/cli_opts_params.yaml" exists
   stat:

tasks/common/convert_nic_templates.yaml

@@ -1,3 +1,4 @@
+---
 - name: look for network software config
   shell: |
     grep "OS::TripleO::.*::Net::SoftwareConfig" {{ item }}

tasks/common/convert_roles_data.yaml

@@ -1,3 +1,4 @@
+---
 - name: Register roles data file location if exists
   shell: "grep '\\-r\\ \\|\\-\\-roles' {{ overcloud_deploy_script }} | awk {'print $2'}"
   register: custom_roles_file

tasks/common/create_l3_agent_connectivity_check_script.yml

@@ -1,3 +1,4 @@
+---
 - block:
     - name: create start l3 agent connectivity check scripts
       template:

tasks/common/create_l3_agent_failover_check_script.yml

@@ -1,3 +1,4 @@
+---
 - block:
     - name: create l3 agent failover check pre script
       template:

tasks/common/create_nova_actions_check_script.yml

@@ -1,3 +1,4 @@
+---
 - name: create post nova actions test
   template:
     src: "nova_actions_check.sh.j2"

tasks/common/create_workload.yml

@@ -16,4 +16,3 @@
     src: "{{ l3_agent_connectivity_check_stop_template }}"
     dest: "{{ l3_agent_connectivity_check_stop_script }}"
     mode: 0775
-

tasks/common/load_roles_from_inventory.yaml

@@ -1,3 +1,4 @@
+---
 - name: generate inventory file
   shell: |
     source {{ undercloud_rc }}

tasks/common/queens_to_rocky_adjust_nic_templates.yaml

@@ -1,3 +1,4 @@
+---
 - name: look for network software config
   shell: |
     grep "OS::TripleO::.*::Net::SoftwareConfig" {{ item }}

tasks/common/ssh_config_skip_host.yml

@@ -1,3 +1,4 @@
+---
 - name: Adjust ssh config to skip host key check
   copy:
     src: ssh_config

tasks/common/undercloud_validate_upgrade.yaml

@@ -20,7 +20,14 @@
   become_user: root
   register: loaded_ovs
 
-- block:
+- when:
+    - not tripleo_ci
+    - >
+      undercloud_reboot or
+      (installed_kernel.stdout != loaded_kernel.stdout) or
+      (installed_ovs.stdout != loaded_ovs.stdout)
+  tags: undercloud_reboot
+  block:
 
     - name: reboot the undercloud
       shell: "sleep 2 && shutdown -r now"
@@ -31,7 +38,7 @@
       become_user: root
 
     - name: wait for node to go down
-        become: no
+      become: false
       command: ping -c1 {{ ansible_host }}
       register: node_down
       until: node_down.rc != 0
@@ -76,6 +83,3 @@
       until: oc_name.stdout.find('{{ overcloud_stack_name }}') != -1
       retries: "{{ service_readiness_count|default(100)|int }}"
       delay: 3
-  when:
-    - not tripleo_ci and (undercloud_reboot or (installed_kernel.stdout != loaded_kernel.stdout) or (installed_ovs.stdout != loaded_ovs.stdout))
-  tags: undercloud_reboot

tasks/fast-forward-upgrade/convert_roles_data.yaml

@@ -1,3 +1,4 @@
+---
 - name: Register roles data file location if exists
   shell: "grep '\\-r\\ \\|\\-\\-roles' {{ overcloud_deploy_script }} | awk {'print $2'}"
   register: custom_roles_file

tasks/fast-forward-upgrade/create-ffu-scripts.yaml

@@ -1,3 +1,4 @@
+---
 - name: create {{ current_release }} undercloud upgrade script
   template:
     src: templates/fast-forward-upgrade/undercloud_upgrade.sh.j2

tasks/fast-forward-upgrade/create-prepare-scripts.yaml

@@ -1,3 +1,4 @@
+---
 - name: create a comptatible deployment scripts from oooq
   include: ../upgrade/use_oooq.yaml
   tags: use_oooq
@@ -5,7 +6,7 @@
 
 - name: make a copy of the initial overcloud deploy script
   copy:
-    remote_src: yes
+    remote_src: true
     src: "{{ overcloud_deploy_script }}"
     dest: "{{ overcloud_deploy_script }}.orig.sh"
 
@@ -56,7 +57,7 @@
     src: "fast-forward-upgrade/overcloud_upgrade_run.sh.j2"
     dest: "{{working_dir}}/overcloud_upgrade_run.sh"
     mode: 0775
-    force: no
+    force: false
 
 - name: create overcloud converge script
   template:

tasks/fast-forward-upgrade/ffu_update_stack_outputs.yaml

@@ -5,7 +5,7 @@
     {{ working_dir }}/ffu_update_stack_outputs.sh 2>&1 {{ timestamper_cmd }} > \
       {{ working_dir }}/ffu_update_stack_outputs.log
   register: ffu_stack_output
-  ignore_errors: yes
+  ignore_errors: true
 
 - name: print stack failures
   shell: |

tasks/fast-forward-upgrade/ffu_upgrade_ceph.yaml

@@ -1,10 +1,11 @@
+---
 - name: upgrade ceph post ffu
   shell: |
     source {{ undercloud_rc }}
     {{ working_dir }}/ffu_upgrade_ceph_script.sh 2>&1 {{ timestamper_cmd }} > \
     {{ working_dir }}/ffu_upgrade_ceph.log
   register: ffu_upgrade_ceph
-  ignore_errors: yes
+  ignore_errors: true
 
 - block:
     - name: print stack failures

tasks/fast-forward-upgrade/ffu_upgrade_converge.yaml

@@ -5,7 +5,7 @@
     {{ working_dir }}/ffu_upgrade_converge_script.sh 2>&1 {{ timestamper_cmd }} > \
       {{ working_dir }}/ffu_upgrade_converge.log
   register: ffu_converge
-  ignore_errors: yes
+  ignore_errors: true
 
 - name: print stack failures
   shell: |

tasks/fast-forward-upgrade/main.yml

@@ -1,3 +1,4 @@
+---
 - name: create upgrade prepare scripts
   include: create-prepare-scripts.yaml
   tags: create_ffu_prepare_scripts
@@ -88,13 +89,14 @@
       tags: ffu_overcloud_upgrade_role
       when: not ffu_bulk|bool
 
-    - block:
+    - when: ceph_osd_enabled|bool
+      block:
         - name: apply pre ffu ceph upgrade workarounds
           command: "{{working_dir}}/pre_ffu_overcloud_ceph_workarounds.sh"
           when: ffu_upgrade_workarounds|bool
           tags: ffu_overcloud_ceph
 
-      - name: run FFU ceph osd nodes ugrade scripts
+        - name: run FFU ceph osd nodes upgrade scripts
           command: "{{ working_dir }}/overcloud_upgrade_CephStorage.sh"
           tags: ffu_overcloud_ceph
 
@@ -109,7 +111,6 @@
             - '../common/l3_agent_connectivity_check_stop_script.yml'
             - '../common/l3_agent_failover_check_post_script.yml'
           tags: ffu_overcloud_ceph
-      when: ceph_osd_enabled|bool
 
     - name: apply pre ffu overcloud converge workarounds
       command: "{{working_dir}}/pre_ffu_overcloud_converge_workarounds.sh"

tasks/fast-forward-upgrade/undercloud-upgrade.yml

@@ -1,3 +1,4 @@
+---
 - name: Run pre undercloud {{ current_release }} upgrade workarounds
   command: "{{ working_dir }}/pre_undercloud_upgrade_{{ current_release }}_workarounds.sh"
   when: ffu_upgrade_workarounds

tasks/update/create-undercloud-update-scripts.yaml

@@ -16,4 +16,3 @@
     - '{{ pre_undercloud_update_workarounds|default([]) }}'
     - '{{ post_undercloud_update_workarounds|default([]) }}'
   when: updates_workarounds|bool
-

tasks/update/overcloud_update_converge.yml

@@ -7,7 +7,7 @@
         bash {{ overcloud_update_converge_script }} 2>&1 {{ timestamper_cmd }} > \
           {{ working_dir }}/overcloud_update_converge.log
   register: overcloud_converge_update
-  ignore_errors: yes
+  ignore_errors: true
 
 - block:
     - name: print stack failures

tasks/update/overcloud_update_prepare.yml

@@ -6,7 +6,7 @@
     bash {{ overcloud_update_prepare_script }} 2>&1 {{ timestamper_cmd }} > \
       {{ working_dir }}/overcloud_update_prepare.log
   register: overcloud_update_prepare
-  ignore_errors: yes
+  ignore_errors: true
 
 - block:
     - name: print stack failures
@@ -26,4 +26,3 @@
     - name: was the overcloud update prepare successful.
       fail: msg="Overcloud minor update preparation step failed..."
   when: overcloud_update_prepare.rc != 0
-

tasks/update/overcloud_update_run.yml

@@ -9,7 +9,7 @@
   with_items:
       - "{{ oc_roles|default('all') }}"
   register: overcloud_update_nodes
-  ignore_errors: yes
+  ignore_errors: true
 
 - name: was the overcloud minor update successful.
   fail: msg="Overcloud minor update execution step failed..."
@@ -17,4 +17,3 @@
   when: item.rc != 0
 
 - include: ../common/l3_agent_connectivity_check_stop_script.yml
-

tasks/upgrade/controller_post_upgrade.yml

@@ -1,3 +1,4 @@
+---
 - name: register controller nodes
   shell: |
     source {{ undercloud_rc }}

tasks/upgrade/create-undercloud-upgrade-scripts.yaml

@@ -15,4 +15,3 @@
     - 'pre_undercloud_upgrade_workarounds'
     - 'post_undercloud_upgrade_workarounds'
   when: upgrade_workarounds
-

tasks/upgrade/kill_ssh.yaml

@@ -1,3 +1,4 @@
+---
 - name: Kill SSH
   shell: sleep 1; pkill -u {{ ansible_ssh_user }} sshd
   async: 3

tasks/upgrade/main.yml

@@ -163,4 +163,3 @@
     - include: ../common/nova_actions_check.yml
 
   when: overcloud_upgrade|bool
-

tasks/upgrade/overcloud_upgrade_converge.yml

@@ -5,7 +5,7 @@
     bash {{ overcloud_upgrade_converge_script }} 2>&1 {{ timestamper_cmd }} > \
         {{ working_dir }}/overcloud_upgrade_converge.log
   register: overcloud_converge_upgrade
-  ignore_errors: yes
+  ignore_errors: true
 
 - block:
     - name: print stack failures

tasks/upgrade/overcloud_upgrade_prepare.yml

@@ -26,7 +26,7 @@
     bash {{ overcloud_upgrade_prepare_script }} 2>&1 {{ timestamper_cmd }} > \
         {{ working_dir }}/overcloud_upgrade_prepare.log
   register: overcloud_upgrade_prepare
-  ignore_errors: yes
+  ignore_errors: true
 
 - block:
     - name: print stack failures

tasks/upgrade/overcloud_upgrade_run.yml

@@ -7,7 +7,7 @@
   with_items:
       - "{{ oc_roles|default('all') }}"
   register: overcloud_upgrade_nodes
-  ignore_errors: yes
+  ignore_errors: true
 
 - name: was the overcloud upgrade composable step successful.
   fail: msg="Overcloud upgrade composable step failed... :("

tasks/upgrade/undercloud_ssl_camap.yaml

@@ -1,3 +1,4 @@
+---
 - name: register undercloud public endpoint
   shell: |
     source {{ undercloud_rc }}
@@ -16,7 +17,7 @@
   shell: |
     ssh -q -o StrictHostKeyChecking=no {{ oc_user }}@{{ ctrl_ip.stdout }} curl --silent {{ keystone_endpoint.stdout }}
   register: uc_keystone_conn
-  ignore_errors: True
+  ignore_errors: true
 
 - block:
     #

test-requirements.txt

@@ -1,6 +1,4 @@
-hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
-bashate>=0.5.1  # Apache-2.0
-
+pre-commit>=1.10 # MIT License
 # this is required for the docs build jobs
 reno>=2.5.0  # Apache-2.0
 sphinx>=1.6.2 # BSD

tests/oooq-test.yaml

@@ -1,7 +1,7 @@
 ---
 - hosts: undercloud
   gather_facts: true
-  become: yes
+  become: true
   become_method: sudo
   become_user: stack
   roles:

tests/test.yml

@@ -1,7 +1,7 @@
 ---
 - hosts: undercloud
   gather_facts: true
-  become: yes
+  become: true
   become_method: sudo
   become_user: stack
   roles:

tox.ini

@@ -24,55 +24,30 @@ basepython = python3
 commands = python setup.py build_sphinx
 
 [testenv:bashate]
+envdir = {toxworkdir}/linters
 commands =
-# Run bashate check for all bash scripts
-# Ignores the following rules:
-# E006: Line longer than 79 columns (as many scripts use jinja
-#       templating, this is very difficult)
-# E040: Syntax error determined using `bash -n` (as many scripts
-#       use jinja templating, this will often fail and the syntax
-#       error will be discovered in execution anyway)
-    bash -c "git ls-files | xargs grep --binary-files=without-match \
-        --files-with-match '^.!.*\(ba\)\?sh$' \
-        --exclude-dir .tox \
-        --exclude-dir .git \
-        | xargs bashate --error . --verbose --ignore=E006,E040"
+    python -m pre_commit run bashate --all-files
 
 [testenv:pep8]
-basepython = python3
+envdir = {toxworkdir}/linters
 commands =
-    # Run hacking/flake8 check for all python files
-    bash -c "git ls-files | grep -v releasenotes |  xargs grep --binary-files=without-match \
-        --files-with-match '^.!.*python$' \
-        --exclude-dir .tox \
-        --exclude-dir .git \
-        --exclude-dir .eggs \
-        --exclude-dir *.egg-info \
-        --exclude-dir dist \
-        --exclude-dir *lib/python* \
-        --exclude-dir doc \
-        | xargs flake8 --verbose"
+    python -m pre_commit run flake8 --all-files
 
 [testenv:ansible-lint]
-basepython=python2
+envdir = {toxworkdir}/linters
 commands =
-  bash ci-scripts/ansible-lint.sh
-
-[testenv:linters]
-basepython = python3
-deps =
-    -r{toxinidir}/test-requirements.txt
-    -r{toxinidir}/ansible-requirements.txt
-commands =
-    {[testenv:bashate]commands}
-    {[testenv:pep8]commands}
-    {[testenv:ansible-lint]commands}
+    python -m pre_commit run ansible-lint -a
 
 [testenv:releasenotes]
 basepython = python3
 whitelist_externals = bash
 commands = bash -c ci-scripts/releasenotes_tox.sh
 
+[testenv:linters]
+commands =
+    # check only modified files:
+    python -m pre_commit run --source HEAD^ --origin HEAD
+
 [testenv:venv]
 basepython = python3
 commands = {posargs}

zuul.d/layout.yaml

@@ -1,3 +1,4 @@
+---
 - project:
     templates:
       - check-requirements