Gate fixes for Shipyard

* Install older version of pip<21.0 for ubuntu_xenial images
* Install setuptools via pip for ubuntu_xenial images
* Pin typing-extensions to 3.7.2 and apache-airflow to 1.10.5
* Move promenade Dependencies under UCP components in requirements.txt
* With apache-airflow=1.10.5, strip ANSI escape sequences
  in test_deployment_group_manager.py
* Update tox.ini to support apache-airflow=1.10.5
* airskiff gate fixes
  - Pin treasuremap to v1.9 branch
  - Pin openstack-helm-infra to master
  - Remove openstack client setup as it's not used

Change-Id: Iee4ce59fdceacb165120a69d11c44e6e47feaea8

.zuul.yaml

@@ -126,6 +126,7 @@
       - airship/treasuremap
     vars:
       CLONE_SHIPYARD: false
+      HTK_COMMIT: master
     irrelevant-files: &irrelevant-files
       - ^.*\.rst$
       - ^doc/.*$

images/airflow/Dockerfile.ubuntu_xenial

@@ -69,13 +69,13 @@ RUN set -ex && \
     netcat \
     netbase \
     python3 \
-    python3-setuptools \
     python3-pip \
     python3-dev \
     python3-dateutil \
     make \
     --no-install-recommends \
-    && python3 -m pip install -U pip \
+    && python3 -m pip install -U 'pip<21.0' \
+    && python3 -m pip install -U setuptools \
     && apt-get clean \
     && rm -rf \
         /var/lib/apt/lists/* \

images/shipyard/Dockerfile.ubuntu_xenial

@@ -46,7 +46,6 @@ RUN set -ex && \
     make \
     netbase \
     python3-dev \
-    python3-setuptools \
     --no-install-recommends \
     && apt-get autoremove -yqq --purge \
     && apt-get clean \
@@ -101,7 +100,8 @@ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
     ' \
     && apt-get -qq update \
     && apt-get -y install -y $buildDeps --no-install-recommends \
-    && python3 -m pip install -U pip \
+    && python3 -m pip install -U 'pip<21.0' \
+    && python3 -m pip install -U setuptools \
     && pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \
     && cd /home/shipyard/shipyard_client \
     && python3 setup.py install \

src/bin/shipyard_airflow/requirements.txt

@@ -58,3 +58,4 @@ tabulate==0.8.03
 git+https://opendev.org/airship/deckhand.git@1f0c011a1708c1235a2be65edada7d386cd55d2a#egg=deckhand
 git+https://opendev.org/airship/drydock.git@96db568d4b28f5f163efbf606854243614a6c821#egg=drydock_provisioner&subdirectory=python
 git+https://opendev.org/airship/armada.git@0c3aff1ba1f0e836e3dd56de81b2ee560e11c179#egg=armada
+git+https://opendev.org/airship/promenade.git@9cd261294bb77c42375ab02606c2f18bff081c26#egg=promenade

src/bin/shipyard_airflow/test-requirements.txt

@@ -5,19 +5,15 @@ pytest==3.5.0
 pytest-cov==2.5.1
 responses==0.10.2
 testfixtures==5.1.1
-# Had to upgrade apache-airflow as 1.10.1 requires a version of jinja too old
-# for promenade
-apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.2
-# install postgres seperately as apache-airflow 1.10.2 forces postgres to use
+typing-extensions==3.7.2
+# Pin apache-airflow 1.10.5 to align with version in airflow images requirements.txt
+apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5
+# install postgres seperately as apache-airflow 1.10.5 forces postgres to use
 # psycopg2 instead of psycopg2-binary which requires additional apt packages
 # to be installed, i.e. postgres-devel. Otherwise the following error is seen:
 # Error: pg_config executable not found. Only seems to be an issue in xenial
 postgres>=3.0.0
 
-# TODO(bryan-strassner) Pin to version for airflow when added to the
-# requirements.txt in the airflow images directory
-git+https://opendev.org/airship/promenade.git@master#egg=promenade
-
 # Linting
 flake8<3.8.0,>=3.6.0
 

src/bin/shipyard_airflow/tests/unit/common/deployment_group/test_deployment_group_manager.py

@@ -15,6 +15,7 @@
 module
 """
 import pytest
+import re
 import yaml
 
 from shipyard_airflow.common.deployment_group.deployment_group import (
@@ -197,13 +198,14 @@ class TestDeploymentGroupManager:
         dgm.mark_group_prepared('compute-nodes-1')
         dgm.mark_group_failed('compute-nodes-2')
         dgm.report_group_summary()
+        ansi_re = re.compile(r'\x1b\[[0-9;]*m')
         assert "=====   Group Summary   =====" in caplog.text
         assert ("Group ntp-node [Critical] ended with stage: "
-                "Stage.NOT_STARTED") in caplog.text
+                "Stage.NOT_STARTED") in re.sub(ansi_re, '', caplog.text)
         caplog.clear()
         dgm.report_node_summary()
-        assert "Nodes Stage.PREPARED: node2" in caplog.text
-        assert "Nodes Stage.FAILED: node3" in caplog.text
+        assert "Nodes Stage.PREPARED: node2" in re.sub(ansi_re, '', caplog.text)
+        assert "Nodes Stage.FAILED: node3" in re.sub(ansi_re, '', caplog.text)
         assert "===== End Node Summary =====" in caplog.text
         assert "It was the best of times" not in caplog.text
 

src/bin/shipyard_airflow/tox.ini

@@ -1,17 +1,21 @@
 [tox]
+skipsdist=True
 envlist = py35, py36, pep8, bandit
 
 [testenv]
 setenv=
   PYTHONWARNING=all
+  LANG=C.UTF-8
+  LC_ALL=C.UTF-8
 deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
 
 [testenv:py35]
 skipsdist=True
 setenv=
   SLUGIFY_USES_TEXT_UNIDECODE=yes
 commands =
+  pip install -r{toxinidir}/test-requirements.txt
+  airflow initdb
   pytest \
     {posargs} \
     --cov-branch \
@@ -25,6 +29,8 @@ skipsdist=True
 setenv=
   SLUGIFY_USES_TEXT_UNIDECODE=yes
 commands =
+  pip install -r{toxinidir}/test-requirements.txt
+  airflow initdb
   pytest \
     {posargs} \
     --cov-branch \
@@ -64,6 +70,7 @@ basepython=python3
 setenv=
   SLUGIFY_USES_TEXT_UNIDECODE=yes
 commands =
+    pip install -r{toxinidir}/test-requirements.txt
     oslo-config-generator --config-file=generator/config-generator.conf
     oslopolicy-sample-generator --config-file=generator/policy-generator.conf
 

src/bin/shipyard_client/tox.ini

@@ -5,6 +5,8 @@ envlist = py35, py36, pep8, bandit
 [testenv]
 setenv=
   PYTHONWARNING=all
+  LC_ALL=C.UTF-8
+  LC_ALL=C.UTF-8
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 

tools/gate/playbooks/airskiff-deploy.yaml

@@ -33,6 +33,7 @@
     - name: Clone Required Repositories
       shell: |
         export CLONE_SHIPYARD={{ CLONE_SHIPYARD }}
+        export OSH_INFRA_COMMIT={{ HTK_COMMIT }}
         ./tools/deployment/airskiff/developer/000-clone-dependencies.sh
       args:
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
@@ -56,13 +57,6 @@
         chdir: "{{ zuul.project.src_dir }}"
       become: yes
 
-    - name: Setup OpenStack Client
-      shell: |
-        ./tools/deployment/airskiff/developer/020-setup-client.sh
-      args:
-        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
-      become: yes
-
     - name: Deploy Airship components using Armada
       shell: |
         mkdir ~/.kube

tools/gate/playbooks/airskiff-reduce-site.yaml

@@ -14,9 +14,23 @@
 
 - hosts: primary
   tasks:
+    - name: ensure pip packages are installed
+      apt:
+        name: "{{ item }}"
+      with_items:
+        - python3-pip
+        - python3-git
+        - python3-requests
+        - python3-yaml
+      become: yes
+
     - name: Overwrite Armada manifest
       shell: |
+        git checkout v1.9
         mv tools/gate/manifests/full-site.yaml \
             type/skiff/manifests/full-site.yaml
+        # Overwrite versions.yaml with the latest Airship charts and xenial images
+        ./tools/updater.py --in-file global/software/config/versions.yaml \
+            --tag-filter ubuntu_xenial --skip mariadb
       args:
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"