Add ability to strip old excludes

There's no point keeping these ancient excludes around: they're not
going to be installed on any modern system. Add a tool to remove these.
Comments are not easily machine parsable and must still be handled
manually, unfortunately.

Change-Id: I9dc1746ca77fa145a8030dbafa9b107872719290
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
This commit is contained in:
Stephen Finucane 2024-04-16 16:40:37 +01:00
parent 258e59dd69
commit 314734e938
2 changed files with 194 additions and 80 deletions

View File

@ -1,14 +1,14 @@
## section:general
alembic!=1.2.0,!=1.6.3 # MIT
amqp!=2.1.4,!=5.0.4 # BSD
ansible-runner!=1.3.5 # Apache 2.0
alembic # MIT
amqp # BSD
ansible-runner # Apache 2.0
appdirs # MIT License
apscheduler # MIT License
autobahn # MIT License
automaton # Apache-2.0
autopage # Apache-2.0
Babel!=2.4.0 # BSD
Babel # BSD
bcrypt # Apache-2.0
beautifulsoup4 # MIT
betamax # Apache-2.0
@ -16,15 +16,15 @@ boto # MIT
boto3 # Apache-2.0
botocore # Apache-2.0
cachetools # MIT License
cassandra-driver!=3.6.0 # Apache-2.0
cassandra-driver # Apache-2.0
cffi # MIT
cmd2!=0.8.3 # MIT
confluent-kafka!=1.4.0 # Apache-2.0
cmd2 # MIT
confluent-kafka # Apache-2.0
confspirator # Apache-2.0
construct # MIT
cotyledon # Apache-2.0
croniter # MIT License
cryptography!=2.0 # BSD/Apache-2.0
cryptography # BSD/Apache-2.0
cursive # Apache-2.0
datetimerange # MIT
decorator # BSD
@ -34,27 +34,25 @@ django-debreach # BSD
django-formtools # BSD
django-pyscss # BSD License (2 clause)
Django<4.3 # BSD
# eventlet is not compatibile with 2.0.0: https://github.com/eventlet/eventlet/issues/619
dnspython!=2.0.0,!=2.2.0 # http://www.dnspython.org/LICENSE
dogpile.cache!=0.9.1,!=1.1.7 # BSD
dnspython # http://www.dnspython.org/LICENSE
dogpile.cache!=1.1.7 # BSD
dogtag-pki # LGPLv3+
dulwich!=0.19.3,!=0.19.7 # Apache-2.0
dulwich # Apache-2.0
edgegrid-python # Apache-2.0
elasticsearch<3.0.0 # Apache-2.0
enmerkar # BSD
# NOTE: New versions of eventlet should not be accepted lightly
# as they have earned a reputation of frequently breaking things.
eventlet!=0.18.3,!=0.20.1,!=0.21.0,!=0.23.0,!=0.25.0,!=0.32.0,!=0.34.1,!=0.34.2,!=0.34.3,!=0.35.0,!=0.36.0 # MIT
exabgp!=4.0.6 # BSD
eventlet!=0.34.1,!=0.34.2,!=0.34.3,!=0.35.0,!=0.36.0 # MIT
exabgp # BSD
falcon # Apache-2.0
# https://github.com/harlowja/fasteners/issues/36
fasteners!=0.15,!=0.16 # Apache-2.0
Flask!=0.11 # BSD
fasteners # Apache-2.0
Flask # BSD
Flask-RESTful # BSD
GitPython # BSD License (3 clause)
google-api-python-client # Apache-2.0
graphviz!=0.5.0 # MIT License
greenlet!=0.4.14 # MIT
graphviz # MIT License
greenlet # MIT
gunicorn # MIT
httplib2 # MIT
httpx # BSD
@ -64,7 +62,7 @@ icalendar # BSD
# newer code than in [most] releases of the Python std library.
importlib-metadata # Apache-2.0
infinisdk # BSD-3
influxdb!=5.3.0 # MIT
influxdb # MIT
influxdb-client # MIT
infoblox-client # Apache-2.0
iso8601 # MIT
@ -72,33 +70,33 @@ jaeger-client # Apache-2.0
Jinja2 # BSD License (3 clause)
jira # BSD License (2 clause)
jmespath # MIT
jsonpatch!=1.20 # BSD
jsonpatch # BSD
jsonschema # MIT
kazoo # Apache-2.0
kombu!=4.0.2 # BSD
kombu # BSD
kubernetes # Apache-2.0
ldap3 # LGPLv3
libvirt-python!=4.1.0,!=4.2.0 # LGPLv2+
lxml!=3.7.0 # BSD
libvirt-python # LGPLv2+
lxml # BSD
Mako # MIT
msgpack # Apache-2.0
munch # MIT
ncclient # Apache-2.0
netaddr # BSD
netifaces!=0.10.0,!=0.10.1 # MIT
netifaces # MIT
netmiko # MIT
networkx!=2.8.4 # BSD
oauthlib # BSD
opentelemetry-exporter-otlp # Apache-2.0
opentelemetry-sdk # Apache-2.0
ovs # Apache-2.0
packaging!=20.5,!=20.6,!=20.7 # Apache-2.0
paramiko!=2.9.0,!=2.9.1 # LGPLv2.1+
packaging # Apache-2.0
paramiko # LGPLv2.1+
passlib # BSD
Paste # MIT
PasteDeploy # MIT
pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,!=1.4.0 # BSD
pexpect!=3.3 # ISC License
pecan # BSD
pexpect # ISC License
pika # BSD
Pillow # PIL License
Pint # BSD
@ -107,27 +105,27 @@ PrettyTable!=3.4.0 # BSD
prometheus-client # Apache-2.0
protobuf # BSD License (3 clause)
psutil # BSD
pyasn1!=0.2.3 # BSD
pyasn1 # BSD
pyasn1-lextudio # BSD
pyasn1-modules # BSD
pyasn1-modules-lextudio # BSD
pycadf!=2.0.0 # Apache-2.0
pycadf # Apache-2.0
pycdlib # LGPLv2+
PyECLib # BSD
pyghmi!=1.4.0,!=1.5.11 # Apache-2.0
pyghmi # Apache-2.0
PyJWT # MIT
pykmip # Apache 2.0 License
pylxd # Apache-2.0
pymemcache!=1.3.0 # Apache 2.0 License
pymemcache # Apache 2.0 License
PyMI;sys_platform=='win32' # Apache 2.0 License
pymongo!=3.1 # Apache-2.0
pymongo # Apache-2.0
PyMySQL # MIT License
pyngus # Apache-2.0
pyOpenSSL # Apache-2.0
pyparsing # MIT
pyroute2!=0.5.4,!=0.5.5,!=0.7.1;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2)
pysaml2!=4.0.3,!=4.0.4,!=4.0.5,!=4.0.5rc1,!=4.1.0,!=4.2.0,!=4.3.0,!=4.4.0,!=4.6.0 # Apache-2.0
pyScss!=1.3.5 # MIT License
pyroute2!=0.7.1;sys_platform != "win32" # Apache-2.0 (+ dual licensed GPL2)
pysaml2 # Apache-2.0
pyScss # MIT License
pysnmp-lextudio # BSD
pystache # MIT
# Only required for sasl/binary protocol
@ -142,7 +140,7 @@ pywinrm # MIT
PyYAML # MIT
pyzabbix # LGPL
rbd-iscsi-client # Apache-2.0
requests!=2.20.0,!=2.24.0 # Apache-2.0
requests # Apache-2.0
requests-aws # BSD License (3 clause)
requests-kerberos # ISC
requestsexceptions # Apache-2.0
@ -150,18 +148,14 @@ rfc3986 # Apache-2.0
Routes # MIT
rtslib-fb # Apache-2.0
ruamel.yaml # MIT
salt!=2019.2.1,!=2019.2.2 # Apache-2.0
salt # Apache-2.0
scikit-learn # BSD
scipy # BSD
# https://github.com/holgern/py-scrypt/issues/16
scrypt!=0.8.21 # BSD
semantic-version # BSD
setproctitle # BSD
# NOTE(yamahata):
# bug work around of sqlalchemy
# https://bitbucket.org/zzzeek/sqlalchemy/issues/3952/
# The fix which is in git master branch is planned for 1.1.9
SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT
SQLAlchemy # MIT
sqlalchemy-filters # Apache-2.0
sqlalchemy-migrate # Apache-2.0
SQLAlchemy-Utils # BSD License
@ -221,7 +215,7 @@ XStatic-Spin # MIT License
XStatic-term.js # MIT License
XStatic-tv4 # MIT
# NOTE(anilvenkata): This is required for profiling oslo.service processes
Yappi!=0.98,!=0.99 # MIT
Yappi # MIT
zeroconf # LGPL
zipp # MIT
zstd # BSD License (2 clause)
@ -230,7 +224,7 @@ zVMCloudConnector;sys_platform!='win32' # Apache 2.0 License
## section:testing
bashate # Apache-2.0
coverage!=4.4 # Apache-2.0
coverage # Apache-2.0
ddt # MIT
django-nose # BSD
docker # Apache-2.0
@ -245,7 +239,7 @@ ldappool # MPL
# Do not make mock conditional on Python version: we depend on newer code than
# in [most] releases of the Python std library.
# https://github.com/testing-cabal/mock/issues/487 for 4.0.[0-1] blacklist
mock!=4.0.0,!=4.0.1 # BSD
mock # BSD
moto # Apache-2.0
mypy # MIT
nodeenv # BSD
@ -267,13 +261,13 @@ pytest-xdist # MIT
python-consul # MIT License
python-subunit # Apache-2.0/BSD
pyzmq # LGPL+BSD
redis!=4.0.0 # MIT
redis # MIT
requests-mock # Apache-2.0
retrying!=1.3.0 # Apache-2.0
retrying # Apache-2.0
sadisplay # BSD
selenium<4.0.0 # Apache-2.0
stestr!=2.3.0,!=3.0.0 # Apache-2.0
sushy!=1.9.0 # Apache-2.0
stestr # Apache-2.0
sushy # Apache-2.0
tabulate # MIT
tenacity # Apache-2.0
testrepository # Apache-2.0/BSD
@ -285,7 +279,7 @@ typing # PSF
typing-extensions # PSF
tzdata # MIT
virtualbmc # Apache-2.0
virtualenv!=16.3.0 # MIT
virtualenv # MIT
WebTest # MIT
Werkzeug!=2.2.0 # BSD License
whereto # Apache-2.0
@ -296,13 +290,13 @@ xvfbwrapper #license: MIT
## section:docs
blockdiag!=2.0.0 # Apache-2.0
blockdiag # Apache-2.0
doc8 # Apache-2.0
pydot # MIT License
pydotplus # MIT License
Pygments # BSD license
rst2txt # BSD
sphinx!=1.6.6,!=1.6.7,!=2.1.0,!=3.0.0,!=3.4.2 # BSD
sphinx # BSD
sphinxcontrib-actdiag # BSD
sphinxcontrib-apidoc # BSD
sphinxcontrib-blockdiag # BSD
@ -327,16 +321,16 @@ ceilometer # Apache-2.0
# ceilometermiddleware might not show up with a search of setup.cfg and
# requirements files, but some projects use it via being installed by devstack
ceilometermiddleware # Apache-2.0
cliff!=2.9.0,!=2.17.0 # Apache-2.0
cliff # Apache-2.0
debtcollector # Apache-2.0
dib-utils # Apache-2.0
diskimage-builder!=1.6.0,!=1.7.0,!=1.7.1 # Apache-2.0
etcd3gw!=0.2.2,!=0.2.3,!=0.2.6 # Apache-2.0
diskimage-builder # Apache-2.0
etcd3gw # Apache-2.0
futurist # Apache-2.0
glance-store!=0.29.0 # Apache-2.0
glance-store # Apache-2.0
heat-translator # Apache-2.0
horizon # Apache-2.0
ironic-lib!=4.6.0 # Apache-2.0
ironic-lib # Apache-2.0
keystoneauth1 # Apache-2.0
keystonemiddleware # Apache-2.0
kuryr-lib # Apache-2.0
@ -357,7 +351,7 @@ neutron-fwaas # Apache-2.0
neutron-lib # Apache-2.0
octavia-lib # Apache-2.0
os-apply-config # Apache-2.0
os-brick!=2.8.0 # Apache-2.0
os-brick # Apache-2.0
os-client-config # Apache-2.0
os-collect-config # Apache-2.0
os-ken # Apache-2.0
@ -365,36 +359,34 @@ os-refresh-config # Apache-2.0
os-resource-classes # Apache-2.0
os-service-types # Apache-2.0
os-traits # Apache-2.0
os-vif!=1.8.0,!=1.12.0,!=3.0.0 # Apache-2.0
os-vif!=3.0.0 # Apache-2.0
os-win # Apache-2.0
osc-lib # Apache-2.0
osc-placement # Apache-2.0
oslo.cache!=1.31.1,!=2.1.0 # Apache-2.0
oslo.cache # Apache-2.0
oslo.concurrency # Apache-2.0
oslo.config!=4.3.0,!=4.4.0 # Apache-2.0
oslo.config # Apache-2.0
oslo.context # Apache-2.0
oslo.db # Apache-2.0
oslo.i18n # Apache-2.0
oslo.limit # Apache-2.0
oslo.log!=3.44.2,!=4.1.2,!=4.2.0,!=5.0.1,!=5.0.2,!=5.1.0 # Apache-2.0
oslo.messaging!=9.0.0 # Apache-2.0
oslo.log!=5.0.1,!=5.0.2,!=5.1.0 # Apache-2.0
oslo.messaging # Apache-2.0
oslo.metrics # Apache-2.0
oslo.middleware # Apache-2.0
oslo.policy!=3.0.0,!=3.6.1 # Apache-2.0
oslo.policy # Apache-2.0
oslo.privsep # Apache-2.0
oslo.reports # Apache-2.0
oslo.rootwrap # Apache-2.0
# NOTE(mriedem): oslo.serialization 2.19.1 is blocked for bug 1593641
oslo.serialization!=2.19.1 # Apache-2.0
oslo.service!=1.28.1 # Apache-2.0
oslo.serialization # Apache-2.0
oslo.service # Apache-2.0
oslo.upgradecheck # Apache-2.0
# NOTE(lajoskatona): oslo.utils version between 3.39.1 and 3.40.1 excluded due to bug 1812922
oslo.utils!=3.39.1,!=3.40.0,!=3.40.1 # Apache-2.0
oslo.utils # Apache-2.0
oslo.versionedobjects # Apache-2.0
oslo.vmware # Apache-2.0
osprofiler # Apache-2.0
pbr!=2.1.0 # Apache-2.0
stevedore!=3.0.0 # Apache-2.0
pbr # Apache-2.0
stevedore # Apache-2.0
tap-as-a-service # Apache-2.0
taskflow # Apache-2.0
tempest # Apache-2.0
@ -413,7 +405,7 @@ gnocchiclient # Apache-2.0
openstacksdk # Apache-2.0
python-barbicanclient # Apache-2.0
python-blazarclient # Apache-2.0
python-cinderclient!=4.0.0 # Apache-2.0
python-cinderclient # Apache-2.0
python-cloudkittyclient # Apache-2.0
python-cyborgclient # Apache-2.0
python-designateclient # Apache-2.0
@ -421,12 +413,12 @@ python-freezerclient # Apache-2.0
python-glanceclient # Apache-2.0
python-heatclient # Apache-2.0
python-ironic-inspector-client # Apache-2.0
python-ironicclient!=2.5.2,!=2.7.1,!=3.0.0 # Apache-2.0
python-keystoneclient!=2.1.0 # Apache-2.0
python-ironicclient # Apache-2.0
python-keystoneclient # Apache-2.0
python-magnumclient # Apache-2.0
python-manilaclient # Apache-2.0
python-masakariclient # Apache-2.0
python-mistralclient!=3.2.0 # Apache-2.0
python-mistralclient # Apache-2.0
python-monascaclient # Apache-2.0
python-muranoclient # Apache-2.0
python-neutronclient # Apache-2.0
@ -449,7 +441,7 @@ python-zunclient # Apache-2.0
##
## Docs-related projects under openstack governance
openstackdocstheme!=2.1.0,!=2.1.1 # Apache-2.0
openstackdocstheme # Apache-2.0
os-api-ref # Apache-2.0
oslosphinx # Apache-2.0
reno # Apache-2.0
@ -479,7 +471,7 @@ python-linstor # LGPLv3
pywbem # LGPLv2.1+
rsd-lib # Apache-2.0
storops # Apache-2.0
storpool!=5.2.0,!=5.3.0 # Apache-2.0
storpool # Apache-2.0
storpool.spopenstack # Apache-2.0
## section:internal
@ -506,7 +498,7 @@ extras # MIT
jsonpath-rw # Apache-2.0
jsonpath-rw-ext # Apache-2.0
kafka-python # Apache-2.0
oauth2client!=4.0.0 # Apache-2.0
oauth2client # Apache-2.0
pyinotify;sys_platform!='win32' and sys_platform!='darwin' and sys_platform!='sunos5' # MIT
# pysnmp library is not maintained since 4 years, it is
# not recommended to use it, use its fork pysnmp-lextudio instead

View File

@ -12,13 +12,20 @@
# License for the specific language governing permissions and limitations
# under the License.
from concurrent import futures
import datetime
import os
import sys
from packaging import requirements
import requests
GLOBAL_REQS = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'..',
'global-requirements.txt',
)
MAX_EXCLUDE_AGE = datetime.timedelta(365 * 2)
def sort() -> None:
@ -75,5 +82,120 @@ def sort() -> None:
fh.write(dep)
def validate_excludes(
name: str, specifiers: requirements.SpecifierSet
) -> tuple[str, str]:
data = requests.get(f'https://pypi.org/pypi/{name}/json').json()
latest_release = max(data['releases'])
result = []
for specifier in specifiers:
if specifier.operator != '!=':
result.append(specifier)
# non-exclusion specifier
continue
exclude = specifier.version
if exclude == latest_release:
print(
f'Release {exclude} is the latest release for package {name}. '
f'Skipping checks.'
)
result.append(specifier)
continue
release = data['releases'].get(exclude)
if not release:
print(
f'Failed to find release {exclude} for package {name}',
file=sys.stderr,
)
continue
if all(r['yanked'] for r in release):
print(f'Release {exclude} for package {name} was yanked')
continue
now = datetime.datetime.now(datetime.timezone.utc)
age = min(
(now - datetime.datetime.fromisoformat(r['upload_time_iso_8601']))
for r in release
)
if age >= MAX_EXCLUDE_AGE:
print(
f'Release {exclude} for package {name} is older than the '
f'upper limit for age '
f'({age.days} days >= {MAX_EXCLUDE_AGE.days} days)'
)
continue
# exclude is recent enough and not yanked so keep it
result.append(specifier)
return name, ','.join(sorted(str(r) for r in result))
def remove_old_excludes():
"""Remove excludes for old package versions.
If we exclude e.g. v1.22 of a package but that version was release over 2
years ago and said package is currently at v1.45, then there's no reason to
keep that exclude around.
"""
deps: dict[str, set[str]] = {}
with open(GLOBAL_REQS) as fh:
for line in fh.readlines():
if not line.strip() or line.startswith('#'):
# ignore blank lines and comments
continue
req = requirements.Requirement(line.split(' #')[0])
if req.name in ('setuptools',):
# ignore certain packages where we want to retain all excludes
continue
# these shouldn't be in our global-requirements file so we don't
# handle them...but make sure
assert not req.extras, f'unexpected extras: {req}'
assert not req.url, f'unexpected url: {req}'
if any(s.operator == '!=' for s in req.specifier):
deps[req.name] = req.specifier
with futures.ThreadPoolExecutor() as executor:
res = executor.map(validate_excludes, *zip(*deps.items()))
deps.update(dict(res))
with open(GLOBAL_REQS) as fh:
data = fh.read()
with open(GLOBAL_REQS, 'w') as fh:
for i, line in enumerate(data.split('\n')):
if i != 0:
fh.write('\n')
if line.startswith('#') or not line.strip():
# skipped (empty or comment)
fh.write(line)
continue
dep, comment, license = line.partition(' #')
req = requirements.Requirement(dep)
if req.name not in deps:
# skipped (no cap)
fh.write(line)
continue
req.specifier = deps[req.name]
# requirements.Requirement.__str__ adds a space after the semicolon
# which we don't want
fh.write(str(req).replace('; ', ';') + comment + license)
if __name__ == '__main__':
remove_old_excludes()
sort()