diff --git a/scripts/ignored_wheels b/scripts/ignored_wheels
new file mode 100644
index 0000000..81a05a3
--- /dev/null
+++ b/scripts/ignored_wheels
@@ -0,0 +1,247 @@
+actdiag
+alabaster
+amqp
+aodhclient
+appdirs
+APScheduler
+asn1crypto
+autobahn
+automaton
+Babel
+bandit
+betamax
+blockdiag
+boto
+botocore
+cachetools
+castellan
+certifi
+chardet
+click
+colorama
+contextlib2
+cotyledon
+CouchDB
+cursive
+daiquiri
+ddt
+debtcollector
+decorator
+dib-utils
+diskimage-builder
+Django
+django-appconf
+django-babel
+django-compressor
+django-formtools
+django-nose
+dnspython
+doc8
+docker
+docker-pycreds
+dogtag-pki
+elasticsearch
+etcd3gw
+eventlet
+extras
+falcon
+fasteners
+fixtures
+Flask
+Flask-RESTful
+Flask-SQLAlchemy
+freezegun
+futurist
+gabbi
+gitdb2
+GitPython
+glance-store
+google-api-python-client
+google-auth
+graphviz
+gunicorn
+icalendar
+idna
+imagesize
+influxdb
+iso8601
+Jinja2
+jmespath
+jsonmodels
+jsonpatch
+jsonpath-rw-ext
+jsonpointer
+jsonschema
+kafka-python
+kazoo
+keyring
+keystoneauth1
+keystonemiddleware
+kombu
+kubernetes
+kuryr-lib
+ldap3
+ldappool
+linecache2
+marathon
+mock
+monasca-common
+monasca-statsd
+monotonic
+mox3
+netaddr
+networkx
+nosexcover
+notifier
+ntlm-auth
+nwdiag
+oauth2client
+openstackdocstheme
+openstack-doc-tools
+openstacksdk
+os-client-config
+oslo.cache
+oslo.concurrency
+oslo.config
+oslo.context
+oslo.db
+oslo.i18n
+oslo.log
+oslo.messaging
+oslo.middleware
+oslo.policy
+oslo.privsep
+oslo.reports
+oslo.rootwrap
+oslo.serialization
+oslo.service
+oslosphinx
+oslotest
+oslo.utils
+oslo.versionedobjects
+oslo.vmware
+os-refresh-config
+packaging
+paramiko
+Parsley
+passlib
+PasteDeploy
+pathlib2
+pbr
+pexpect
+pifpaf
+pika
+progressbar2
+ptyprocess
+py
+pyasn1
+pyasn1-modules
+Pygments
+PyJWT
+pylev
+pymemcache
+pymod2pkg
+PyMySQL
+pyOpenSSL
+pyparsing
+pypowervm
+pysmi
+pysnmp
+pytest
+python-ceilometerclient
+python-cinderclient
+python-consul
+python-dateutil
+python-designateclient
+python-glanceclient
+python-glareclient
+python-heatclient
+python-ironicclient
+python-karborclient
+python-keystoneclient
+python-magnumclient
+python-manilaclient
+python-mimeparse
+python-monascaclient
+python-muranoclient
+python-neutronclient
+python-novaclient
+python-openstackclient
+python-saharaclient
+python-searchlightclient
+python-senlinclient
+python-smaugclient
+python-solumclient
+python-subunit
+python-swiftclient
+python-tackerclient
+python-troveclient
+python-utils
+python-watcherclient
+python-zaqarclient
+pytz
+redis
+requests
+requestsexceptions
+requests-kerberos
+requests-mock
+requests-ntlm
+requests-toolbelt
+rfc3986
+Routes
+rsa
+sadisplay
+scp
+selenium
+seqdiag
+six
+smmap2
+snowballstemmer
+Sphinx
+sphinxcontrib-actdiag
+sphinxcontrib-blockdiag
+sphinxcontrib-httpdomain
+sphinxcontrib-nwdiag
+sphinxcontrib-seqdiag
+sphinxcontrib-websupport
+sphinxmark
+sphinx-testing
+sqlparse
+statsd
+stevedore
+taskflow
+tempest
+testresources
+testscenarios
+testtools
+tooz
+traceback2
+tricircleclient
+txaio
+unittest2
+uritemplate
+urllib3
+vine
+virtualenv
+waitress
+wcwidth
+WebOb
+websocket-client
+WebTest
+Werkzeug
+xlrd
+xlwt
+xmltodict
+XStatic-Angular
+XStatic-Angular-Bootstrap
+XStatic-Angular-Gettext
+XStatic-angular-ui-router
+XStatic-Bootstrap-SCSS
+XStatic-bootswatch
+XStatic-D3
+XStatic-Font-Awesome
+XStatic-JSEncrypt
+XStatic-roboto-fontface
+XStatic-smart-table
+zake
+zhmcclient
diff --git a/scripts/ignored_wheels_py2 b/scripts/ignored_wheels_py2
new file mode 100644
index 0000000..3354dc5
--- /dev/null
+++ b/scripts/ignored_wheels_py2
@@ -0,0 +1,85 @@
+bashate
+bcrypt
+beautifulsoup4
+betamax-matchers
+ceilometermiddleware
+cffi
+cliff
+coverage
+cryptography
+django-openstack-auth
+docutils
+enum34
+Flask
+funcsigs
+futures
+gevent
+greenlet
+grpcio
+heat-translator
+instack-undercloud
+ipaddress
+ironic-lib
+lxml
+microversion-parse
+mistral-lib
+murano-pkg-check
+ndg-httpsclient
+neutron-lib
+nose
+numpy
+openstack-release-test
+os-api-ref
+os-apply-config
+os-brick
+osc-lib
+os-collect-config
+os-dpm
+os-net-config
+osprofiler
+os-service-types
+os-testr
+os-traits
+os-vif
+os-win
+os-xenapi
+ovsdbapp
+Paste
+paunch
+pecan
+Pillow
+prompt-toolkit
+protobuf
+psycopg2
+pycadf
+pyghmi
+pymongo
+PyNaCl
+python-barbicanclient
+python-congressclient
+python-freezerclient
+python-hnvclient
+python-ironic-inspector-client
+python-kingbirdclient
+python-masakariclient
+python-mistralclient
+python-octaviaclient
+python-vitrageclient
+python-zunclient
+pyzmq
+reno
+repoze.who
+scikit-learn
+scipy
+shade
+singledispatch
+sqlalchemy-migrate
+sushy
+tempest-lib
+tosca-parser
+tripleo-common
+typing
+virtualbmc
+WSME
+yaql
+zope.interface
diff --git a/scripts/ignored_wheels_py3 b/scripts/ignored_wheels_py3
new file mode 100644
index 0000000..2127fd7
--- /dev/null
+++ b/scripts/ignored_wheels_py3
@@ -0,0 +1,46 @@
+argh
+backports-abc
+bcrypt
+beautifulsoup4
+brotlipy
+cffi
+click-repl
+cliff-tablib
+coverage
+cryptography
+cssutils
+deprecation
+docutils
+EditorConfig
+enum34
+exabgp
+flask-keystone
+gevent
+greenlet
+grpcio
+h2
+hpack
+hyperframe
+lxml
+mitmproxy
+mypy
+nose
+numpy
+Paste
+Pillow
+prompt-toolkit
+protobuf
+psycopg2
+pymongo
+PyNaCl
+pyzmq
+repoze.lru
+repoze.who
+ruamel.yaml
+scikit-learn
+scipy
+semantic-version
+sortedcontainers
+typed-ast
+typing
+zope.interface
diff --git a/scripts/requirements.sh b/scripts/requirements.sh
index d1c8949..7b56a0c 100755
--- a/scripts/requirements.sh
+++ b/scripts/requirements.sh
@@ -14,23 +14,34 @@ if (( $(openssl version | awk -F'[ .]' '{print $3}') >= 1 )); then
     sed -i '/python-qpid-proton/d' /upper-constraints.txt
 fi
 
+if [[ "${PYTHON3}" == "no" ]]; then
+    ignore_wheels=py2
+else
+    ignore_wheels=py3
+fi
+
+pushd $(mktemp -d)
+# NOTE(SamYaple): Exclude known native wheels from list
+cp /upper-constraints.txt tmp-upper-constraints.txt
+cat $(dirname $0)/ignored_wheels{,_${ignore_wheels}} | xargs -n1 -I{} sed -i '/^{}=/d' tmp-upper-constraints.txt
+
 # NOTE(SamYaple): Build all deps in parallel. This is safe because we are
 # constrained on the version and we are building with --no-deps
 export CASS_DRIVER_BUILD_CONCURRENCY=8
-pushd $(mktemp -d)
-split -l1 /upper-constraints.txt
+split -l1 tmp-upper-constraints.txt
 ls -1 | xargs -n1 -P20 -t pip wheel --no-deps --wheel-dir / -c /upper-constraints.txt -r | tee /tmp/wheels.txt
 popd
 # NOTE(SamYaple): Handle packages not in upper-constriants and not in PyPI as
 # native whls
 additional_packages=(git+https://github.com/openstack-infra/bindep@24427065c5f30047ac80370be0a390e7f417ce34 uwsgi)
-echo "${additional_packages[@]}" | xargs -n1 -P20 pip wheel --wheel-dir / -c /upper-constraints.txt
+echo "${additional_packages[@]}" | xargs -n1 -P20 pip wheel --wheel-dir / -c /upper-constraints.txt | tee -a /tmp/wheels.txt
 
 # NOTE(SamYaple) Remove native-binary wheels, we only want to keep wheels that
-# we compiled ourselves
-# TODO(SamYaple): Figure out how to not download a whl if it already exists in
-# whl form upstream
-awk -F'[ ,]+' '/^Skipping/ {gsub("-","_");print $2}' /tmp/wheels.txt | xargs -n1 bash -c 'ls /$1-*' _ | xargs rm
+# we compiled ourselves. We should not have any of these, but as packages
+# convert to wheels overtime some will slip through
+# TODO(SamYaple): Periodically check that no additional wheels are
+# inappropriately downloaded
+awk -F'[ ,]+' '/^Skipping/ {gsub("-","_");print $2}' /tmp/wheels.txt | xargs -r -n1 bash -c 'ls /$1-*' _ | sort -u | xargs -t -r rm
 
 # NOTE(SamYaple): We want to purge all files that are not wheels or txt to
 # reduce the size of the layer to only what is needed