Fix requirements building

We were not failing when wheels failed to build. Implement a quick fix
for that issue

Change-Id: I934beb526caca90db93c01c8ec5ac9dd63d6ddc9

Dockerfile

@@ -8,7 +8,7 @@ ARG PROJECT_REPO=https://git.openstack.org/openstack/${PROJECT}
 ARG PROJECT_REF=master
 ARG DISTRO
 ARG PROFILES
-ARG PIP_PACKAGES
+ARG PIP_PACKAGES=""
 ARG PLUGIN=no
 ARG PYTHON3=no
 

bindep.txt

@@ -30,6 +30,7 @@ libldap2-dev                [platform:dpkg requirements]
 libmariadbclient-dev        [platform:debian requirements]
 libmysqlclient-dev          [platform:ubuntu requirements]
 libnss3-dev                 [platform:dpkg requirements]
+libpcre3-dev                [platform:dpkg requirements]
 libpq-dev                   [platform:dpkg requirements]
 libpython2.7                [platform:dpkg !python3]
 libpython3.5                [platform:dpkg python3]
@@ -59,6 +60,7 @@ openldap-devel              [platform:rpm requirements]
 openssl-devel               [platform:rpm requirements]
 openvswitch                 [platform:rpm (neutron openvswitch) (nova openvswitch)]
 openvswitch-switch          [platform:dpkg (neutron openvswitch) (nova openvswitch)]
+pcre-devel                  [platform:rpm requirements]
 pkg-config                  [platform:dpkg requirements]
 pkgconfig                   [platform:rpm requirements]
 postgresql-devel            [platform:rpm requirements]

playbooks/loci-builder.yaml

@@ -10,7 +10,7 @@
       block:
         - docker_image:
             name: openstackloci/requirements
-            tag: "{{ branch}}-{{ item.name }}"
+            tag: "{{ branch }}-{{ item.name }}"
             repository: 172.17.0.1:5000/openstackloci/requirements
             push: yes
           with_items: "{{ distros }}"

scripts/install.sh

@@ -60,6 +60,7 @@ fi
 
 # NOTE(SamYaple): Remove when bindep>2.5.0 is released
 patch /var/lib/openstack/lib/python*/site-packages/bindep/depends.py < /opt/loci/scripts/bindep.depends.patch
+rm -f /var/lib/openstack/lib/python*/site-packages/bindep/depends.pyc
 
 $(dirname $0)/clone_project.sh
 $(dirname $0)/pip_install.sh /tmp/${PROJECT} ${PIP_PACKAGES}

scripts/requirements.sh

@@ -22,6 +22,7 @@ $(dirname $0)/setup_pip.sh
 pip install bindep==2.5.0
 # NOTE(SamYaple): Remove when bindep>2.5.0 is released
 patch /var/lib/openstack/lib/python*/site-packages/bindep/depends.py < /opt/loci/scripts/bindep.depends.patch
+rm -f /var/lib/openstack/lib/python*/site-packages/bindep/depends.pyc
 
 $(dirname $0)/install_packages.sh
 $(dirname $0)/clone_project.sh
@@ -48,12 +49,15 @@ cat $(dirname $0)/ignored_wheels{,_${ignore_wheels}} | xargs -n1 -I{} sed -i '/^
 # constrained on the version and we are building with --no-deps
 export CASS_DRIVER_BUILD_CONCURRENCY=8
 split -l1 tmp-upper-constraints.txt
-ls -1 | xargs -n1 -P20 -t pip wheel --no-deps --wheel-dir / -c /upper-constraints.txt -r | tee /tmp/wheels.txt
-popd
-# NOTE(SamYaple): Handle packages not in upper-constriants and not in PyPI as
-# native whls
-additional_packages=(uwsgi)
-echo "${additional_packages[@]}" | xargs -n1 -P20 pip wheel --wheel-dir / -c /upper-constraints.txt | tee -a /tmp/wheels.txt
+echo uwsgi ${PIP_PACKAGES} | xargs -n1 | split -l1 -a3
+ls -1 | xargs -n1 -P20 -t bash -c 'pip wheel --no-deps --wheel-dir / -c /upper-constraints.txt -r $1 || echo %1 >> /failure' _ | tee /tmp/wheels.txt
+
+# TODO(SamYaple): Improve the failure catching
+if [[ -f /failure ]]; then
+    echo Wheel failed to build
+    cat /failure
+    exit 1
+fi
 
 # NOTE(SamYaple) Remove native-binary wheels, we only want to keep wheels that
 # we compiled ourselves. We should not have any of these, but as packages