From e074cdf6d9f6504274818f4ca8f43ffab13b2411 Mon Sep 17 00:00:00 2001
From: Sam Yaple <sam@yaple.net>
Date: Thu, 19 Oct 2017 18:35:53 -0400
Subject: [PATCH] Only save compiled wheels

With this patch we stop trying to collect all possible packages we would
want and instead only save packages that we needed to build into wheels
ourselves because they were only distrubuted as source tarballs upstream

Doing it this way has a few advantages:
  * Reduced layer size to 50MB wheels from 250MB wheels
  * Can install additional packages that were not built into the wheels
image ahead of time (assuming they exist as wheels upstream)
  * Faster build times on slower WAN connections
  * Less I/O when building

We explicitly limit to binary installs with "--only-binary :all:" to
prevent any accidental source building.

This has no affect on building in an air-gapped environment. You would
already need a PyPI mirror to build in an air-gapped environment in the
first place.

Change-Id: I746d315ff642c951bbe798af7fbf951b5aa81a4f
---
 scripts/install.sh      |  2 +-
 scripts/pip_install.sh  |  2 +-
 scripts/requirements.sh | 13 ++++++++++---
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/scripts/install.sh b/scripts/install.sh
index e29c2bf..6cf08f6 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -43,7 +43,7 @@ if [[ "${PLUGIN}" == "no" ]]; then
     $(dirname $0)/create_user.sh
     $(dirname $0)/setup_pip.sh
     $(dirname $0)/pip_install.sh \
-        bindep \
+        bindep==2.5.1.dev1 \
         pycrypto \
         pymysql \
         python-memcached \
diff --git a/scripts/pip_install.sh b/scripts/pip_install.sh
index 067f1e1..6c36b3a 100755
--- a/scripts/pip_install.sh
+++ b/scripts/pip_install.sh
@@ -4,4 +4,4 @@ set -ex
 
 packages=$@
 
-pip install --no-cache-dir --no-index --no-compile --upgrade --find-links /tmp/wheels/ ${packages}
+pip install --no-cache-dir --only-binary :all: --no-compile -c /tmp/wheels/upper-constraints.txt --find-links /tmp/wheels/ ${packages}
diff --git a/scripts/requirements.sh b/scripts/requirements.sh
index fd1a6b1..684d4bb 100755
--- a/scripts/requirements.sh
+++ b/scripts/requirements.sh
@@ -12,12 +12,19 @@ mv /tmp/requirements/{global-requirements.txt,upper-constraints.txt} /
 # constrained on the version and we are building with --no-deps
 pushd $(mktemp -d)
 split -l1 /upper-constraints.txt
-ls -1 | xargs -n1 -P20 -t pip wheel --no-deps --wheel-dir / -c /upper-constraints.txt -r
+ls -1 | xargs -n1 -P20 -t pip wheel --no-deps --wheel-dir / -c /upper-constraints.txt -r | tee /tmp/wheels.txt
 popd
-# NOTE(SamYaple): Handle packages not in global-requirements
-additional_packages=(argparse git+https://github.com/openstack-infra/bindep pip setuptools uwsgi wheel virtualenv)
+# NOTE(SamYaple): Handle packages not in upper-constriants and not in PyPI as
+# native whls
+additional_packages=(git+https://github.com/openstack-infra/bindep@24427065c5f30047ac80370be0a390e7f417ce34 uwsgi)
 echo "${additional_packages[@]}" | xargs -n1 -P20 pip wheel --wheel-dir / -c /upper-constraints.txt
 
+# NOTE(SamYaple) Remove native-binary wheels, we only want to keep wheels that
+# we compiled ourselves
+# TODO(SamYaple): Figure out how to not download a whl if it already exists in
+# whl form upstream
+awk -F'[ ,]+' '/^Skipping/ {gsub("-","_");print $2}' /tmp/wheels.txt | xargs -n1 bash -c 'ls /$1-*' _ | xargs rm
+
 # NOTE(SamYaple): We want to purge all files that are not wheels or txt to
 # reduce the size of the layer to only what is needed
 shopt -s extglob