Merge "Ignore wheels that exist natively whl upstream"
This commit is contained in:
Zuul
Gerrit Code Review
commit
fc75bb8bf4
|
|
@ -0,0 +1,247 @@
|
|||
actdiag
|
||||
alabaster
|
||||
amqp
|
||||
aodhclient
|
||||
appdirs
|
||||
APScheduler
|
||||
asn1crypto
|
||||
autobahn
|
||||
automaton
|
||||
Babel
|
||||
bandit
|
||||
betamax
|
||||
blockdiag
|
||||
boto
|
||||
botocore
|
||||
cachetools
|
||||
castellan
|
||||
certifi
|
||||
chardet
|
||||
click
|
||||
colorama
|
||||
contextlib2
|
||||
cotyledon
|
||||
CouchDB
|
||||
cursive
|
||||
daiquiri
|
||||
ddt
|
||||
debtcollector
|
||||
decorator
|
||||
dib-utils
|
||||
diskimage-builder
|
||||
Django
|
||||
django-appconf
|
||||
django-babel
|
||||
django-compressor
|
||||
django-formtools
|
||||
django-nose
|
||||
dnspython
|
||||
doc8
|
||||
docker
|
||||
docker-pycreds
|
||||
dogtag-pki
|
||||
elasticsearch
|
||||
etcd3gw
|
||||
eventlet
|
||||
extras
|
||||
falcon
|
||||
fasteners
|
||||
fixtures
|
||||
Flask
|
||||
Flask-RESTful
|
||||
Flask-SQLAlchemy
|
||||
freezegun
|
||||
futurist
|
||||
gabbi
|
||||
gitdb2
|
||||
GitPython
|
||||
glance-store
|
||||
google-api-python-client
|
||||
google-auth
|
||||
graphviz
|
||||
gunicorn
|
||||
icalendar
|
||||
idna
|
||||
imagesize
|
||||
influxdb
|
||||
iso8601
|
||||
Jinja2
|
||||
jmespath
|
||||
jsonmodels
|
||||
jsonpatch
|
||||
jsonpath-rw-ext
|
||||
jsonpointer
|
||||
jsonschema
|
||||
kafka-python
|
||||
kazoo
|
||||
keyring
|
||||
keystoneauth1
|
||||
keystonemiddleware
|
||||
kombu
|
||||
kubernetes
|
||||
kuryr-lib
|
||||
ldap3
|
||||
ldappool
|
||||
linecache2
|
||||
marathon
|
||||
mock
|
||||
monasca-common
|
||||
monasca-statsd
|
||||
monotonic
|
||||
mox3
|
||||
netaddr
|
||||
networkx
|
||||
nosexcover
|
||||
notifier
|
||||
ntlm-auth
|
||||
nwdiag
|
||||
oauth2client
|
||||
openstackdocstheme
|
||||
openstack-doc-tools
|
||||
openstacksdk
|
||||
os-client-config
|
||||
oslo.cache
|
||||
oslo.concurrency
|
||||
oslo.config
|
||||
oslo.context
|
||||
oslo.db
|
||||
oslo.i18n
|
||||
oslo.log
|
||||
oslo.messaging
|
||||
oslo.middleware
|
||||
oslo.policy
|
||||
oslo.privsep
|
||||
oslo.reports
|
||||
oslo.rootwrap
|
||||
oslo.serialization
|
||||
oslo.service
|
||||
oslosphinx
|
||||
oslotest
|
||||
oslo.utils
|
||||
oslo.versionedobjects
|
||||
oslo.vmware
|
||||
os-refresh-config
|
||||
packaging
|
||||
paramiko
|
||||
Parsley
|
||||
passlib
|
||||
PasteDeploy
|
||||
pathlib2
|
||||
pbr
|
||||
pexpect
|
||||
pifpaf
|
||||
pika
|
||||
progressbar2
|
||||
ptyprocess
|
||||
py
|
||||
pyasn1
|
||||
pyasn1-modules
|
||||
Pygments
|
||||
PyJWT
|
||||
pylev
|
||||
pymemcache
|
||||
pymod2pkg
|
||||
PyMySQL
|
||||
pyOpenSSL
|
||||
pyparsing
|
||||
pypowervm
|
||||
pysmi
|
||||
pysnmp
|
||||
pytest
|
||||
python-ceilometerclient
|
||||
python-cinderclient
|
||||
python-consul
|
||||
python-dateutil
|
||||
python-designateclient
|
||||
python-glanceclient
|
||||
python-glareclient
|
||||
python-heatclient
|
||||
python-ironicclient
|
||||
python-karborclient
|
||||
python-keystoneclient
|
||||
python-magnumclient
|
||||
python-manilaclient
|
||||
python-mimeparse
|
||||
python-monascaclient
|
||||
python-muranoclient
|
||||
python-neutronclient
|
||||
python-novaclient
|
||||
python-openstackclient
|
||||
python-saharaclient
|
||||
python-searchlightclient
|
||||
python-senlinclient
|
||||
python-smaugclient
|
||||
python-solumclient
|
||||
python-subunit
|
||||
python-swiftclient
|
||||
python-tackerclient
|
||||
python-troveclient
|
||||
python-utils
|
||||
python-watcherclient
|
||||
python-zaqarclient
|
||||
pytz
|
||||
redis
|
||||
requests
|
||||
requestsexceptions
|
||||
requests-kerberos
|
||||
requests-mock
|
||||
requests-ntlm
|
||||
requests-toolbelt
|
||||
rfc3986
|
||||
Routes
|
||||
rsa
|
||||
sadisplay
|
||||
scp
|
||||
selenium
|
||||
seqdiag
|
||||
six
|
||||
smmap2
|
||||
snowballstemmer
|
||||
Sphinx
|
||||
sphinxcontrib-actdiag
|
||||
sphinxcontrib-blockdiag
|
||||
sphinxcontrib-httpdomain
|
||||
sphinxcontrib-nwdiag
|
||||
sphinxcontrib-seqdiag
|
||||
sphinxcontrib-websupport
|
||||
sphinxmark
|
||||
sphinx-testing
|
||||
sqlparse
|
||||
statsd
|
||||
stevedore
|
||||
taskflow
|
||||
tempest
|
||||
testresources
|
||||
testscenarios
|
||||
testtools
|
||||
tooz
|
||||
traceback2
|
||||
tricircleclient
|
||||
txaio
|
||||
unittest2
|
||||
uritemplate
|
||||
urllib3
|
||||
vine
|
||||
virtualenv
|
||||
waitress
|
||||
wcwidth
|
||||
WebOb
|
||||
websocket-client
|
||||
WebTest
|
||||
Werkzeug
|
||||
xlrd
|
||||
xlwt
|
||||
xmltodict
|
||||
XStatic-Angular
|
||||
XStatic-Angular-Bootstrap
|
||||
XStatic-Angular-Gettext
|
||||
XStatic-angular-ui-router
|
||||
XStatic-Bootstrap-SCSS
|
||||
XStatic-bootswatch
|
||||
XStatic-D3
|
||||
XStatic-Font-Awesome
|
||||
XStatic-JSEncrypt
|
||||
XStatic-roboto-fontface
|
||||
XStatic-smart-table
|
||||
zake
|
||||
zhmcclient
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
bashate
|
||||
bcrypt
|
||||
beautifulsoup4
|
||||
betamax-matchers
|
||||
ceilometermiddleware
|
||||
cffi
|
||||
cliff
|
||||
coverage
|
||||
cryptography
|
||||
django-openstack-auth
|
||||
docutils
|
||||
enum34
|
||||
Flask
|
||||
funcsigs
|
||||
futures
|
||||
gevent
|
||||
greenlet
|
||||
grpcio
|
||||
heat-translator
|
||||
instack-undercloud
|
||||
ipaddress
|
||||
ironic-lib
|
||||
lxml
|
||||
microversion-parse
|
||||
mistral-lib
|
||||
murano-pkg-check
|
||||
ndg-httpsclient
|
||||
neutron-lib
|
||||
nose
|
||||
numpy
|
||||
openstack-release-test
|
||||
os-api-ref
|
||||
os-apply-config
|
||||
os-brick
|
||||
osc-lib
|
||||
os-collect-config
|
||||
os-dpm
|
||||
os-net-config
|
||||
osprofiler
|
||||
os-service-types
|
||||
os-testr
|
||||
os-traits
|
||||
os-vif
|
||||
os-win
|
||||
os-xenapi
|
||||
ovsdbapp
|
||||
Paste
|
||||
paunch
|
||||
pecan
|
||||
Pillow
|
||||
prompt-toolkit
|
||||
protobuf
|
||||
psycopg2
|
||||
pycadf
|
||||
pyghmi
|
||||
pymongo
|
||||
PyNaCl
|
||||
python-barbicanclient
|
||||
python-congressclient
|
||||
python-freezerclient
|
||||
python-hnvclient
|
||||
python-ironic-inspector-client
|
||||
python-kingbirdclient
|
||||
python-masakariclient
|
||||
python-mistralclient
|
||||
python-octaviaclient
|
||||
python-vitrageclient
|
||||
python-zunclient
|
||||
pyzmq
|
||||
reno
|
||||
repoze.who
|
||||
scikit-learn
|
||||
scipy
|
||||
shade
|
||||
singledispatch
|
||||
sqlalchemy-migrate
|
||||
sushy
|
||||
tempest-lib
|
||||
tosca-parser
|
||||
tripleo-common
|
||||
typing
|
||||
virtualbmc
|
||||
WSME
|
||||
yaql
|
||||
zope.interface
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
argh
|
||||
backports-abc
|
||||
bcrypt
|
||||
beautifulsoup4
|
||||
brotlipy
|
||||
cffi
|
||||
click-repl
|
||||
cliff-tablib
|
||||
coverage
|
||||
cryptography
|
||||
cssutils
|
||||
deprecation
|
||||
docutils
|
||||
EditorConfig
|
||||
enum34
|
||||
exabgp
|
||||
flask-keystone
|
||||
gevent
|
||||
greenlet
|
||||
grpcio
|
||||
h2
|
||||
hpack
|
||||
hyperframe
|
||||
lxml
|
||||
mitmproxy
|
||||
mypy
|
||||
nose
|
||||
numpy
|
||||
Paste
|
||||
Pillow
|
||||
prompt-toolkit
|
||||
protobuf
|
||||
psycopg2
|
||||
pymongo
|
||||
PyNaCl
|
||||
pyzmq
|
||||
repoze.lru
|
||||
repoze.who
|
||||
ruamel.yaml
|
||||
scikit-learn
|
||||
scipy
|
||||
semantic-version
|
||||
sortedcontainers
|
||||
typed-ast
|
||||
typing
|
||||
zope.interface
|
||||
|
|
@ -14,23 +14,34 @@ if (( $(openssl version | awk -F'[ .]' '{print $3}') >= 1 )); then
|
|||
sed -i '/python-qpid-proton/d' /upper-constraints.txt
|
||||
fi
|
||||
|
||||
if [[ "${PYTHON3}" == "no" ]]; then
|
||||
ignore_wheels=py2
|
||||
else
|
||||
ignore_wheels=py3
|
||||
fi
|
||||
|
||||
pushd $(mktemp -d)
|
||||
# NOTE(SamYaple): Exclude known native wheels from list
|
||||
cp /upper-constraints.txt tmp-upper-constraints.txt
|
||||
cat $(dirname $0)/ignored_wheels{,_${ignore_wheels}} | xargs -n1 -I{} sed -i '/^{}=/d' tmp-upper-constraints.txt
|
||||
|
||||
# NOTE(SamYaple): Build all deps in parallel. This is safe because we are
|
||||
# constrained on the version and we are building with --no-deps
|
||||
export CASS_DRIVER_BUILD_CONCURRENCY=8
|
||||
pushd $(mktemp -d)
|
||||
split -l1 /upper-constraints.txt
|
||||
split -l1 tmp-upper-constraints.txt
|
||||
ls -1 | xargs -n1 -P20 -t pip wheel --no-deps --wheel-dir / -c /upper-constraints.txt -r | tee /tmp/wheels.txt
|
||||
popd
|
||||
# NOTE(SamYaple): Handle packages not in upper-constriants and not in PyPI as
|
||||
# native whls
|
||||
additional_packages=(git+https://github.com/openstack-infra/bindep@24427065c5f30047ac80370be0a390e7f417ce34 uwsgi)
|
||||
echo "${additional_packages[@]}" | xargs -n1 -P20 pip wheel --wheel-dir / -c /upper-constraints.txt
|
||||
echo "${additional_packages[@]}" | xargs -n1 -P20 pip wheel --wheel-dir / -c /upper-constraints.txt | tee -a /tmp/wheels.txt
|
||||
|
||||
# NOTE(SamYaple) Remove native-binary wheels, we only want to keep wheels that
|
||||
# we compiled ourselves
|
||||
# TODO(SamYaple): Figure out how to not download a whl if it already exists in
|
||||
# whl form upstream
|
||||
awk -F'[ ,]+' '/^Skipping/ {gsub("-","_");print $2}' /tmp/wheels.txt | xargs -n1 bash -c 'ls /$1-*' _ | xargs rm
|
||||
# we compiled ourselves. We should not have any of these, but as packages
|
||||
# convert to wheels overtime some will slip through
|
||||
# TODO(SamYaple): Periodically check that no additional wheels are
|
||||
# inappropriately downloaded
|
||||
awk -F'[ ,]+' '/^Skipping/ {gsub("-","_");print $2}' /tmp/wheels.txt | xargs -r -n1 bash -c 'ls /$1-*' _ | sort -u | xargs -t -r rm
|
||||
|
||||
# NOTE(SamYaple): We want to purge all files that are not wheels or txt to
|
||||
# reduce the size of the layer to only what is needed
|
||||
|
|
|
|||
Loading…
Reference in New Issue