From 314734e938f107cbd5ebcc7af4d9167c11347406 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Tue, 16 Apr 2024 16:40:37 +0100 Subject: [PATCH] Add ability to strip old excludes There's no point keeping these ancient excludes around: they're not going to be installed on any modern system. Add a tool to remove these. Comments are not easily machine parsable and must still be handled manually, unfortunately. Change-Id: I9dc1746ca77fa145a8030dbafa9b107872719290 Signed-off-by: Stephen Finucane --- global-requirements.txt | 152 +++++++++++++++++++--------------------- tools/lint.py | 122 ++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 80 deletions(-) diff --git a/global-requirements.txt b/global-requirements.txt index e1011e4578..d31d141e91 100644 --- a/global-requirements.txt +++ b/global-requirements.txt @@ -1,14 +1,14 @@ ## section:general -alembic!=1.2.0,!=1.6.3 # MIT -amqp!=2.1.4,!=5.0.4 # BSD -ansible-runner!=1.3.5 # Apache 2.0 +alembic # MIT +amqp # BSD +ansible-runner # Apache 2.0 appdirs # MIT License apscheduler # MIT License autobahn # MIT License automaton # Apache-2.0 autopage # Apache-2.0 -Babel!=2.4.0 # BSD +Babel # BSD bcrypt # Apache-2.0 beautifulsoup4 # MIT betamax # Apache-2.0 @@ -16,15 +16,15 @@ boto # MIT boto3 # Apache-2.0 botocore # Apache-2.0 cachetools # MIT License -cassandra-driver!=3.6.0 # Apache-2.0 +cassandra-driver # Apache-2.0 cffi # MIT -cmd2!=0.8.3 # MIT -confluent-kafka!=1.4.0 # Apache-2.0 +cmd2 # MIT +confluent-kafka # Apache-2.0 confspirator # Apache-2.0 construct # MIT cotyledon # Apache-2.0 croniter # MIT License -cryptography!=2.0 # BSD/Apache-2.0 +cryptography # BSD/Apache-2.0 cursive # Apache-2.0 datetimerange # MIT decorator # BSD @@ -34,27 +34,25 @@ django-debreach # BSD django-formtools # BSD django-pyscss # BSD License (2 clause) Django<4.3 # BSD -# eventlet is not compatibile with 2.0.0: https://github.com/eventlet/eventlet/issues/619 -dnspython!=2.0.0,!=2.2.0 # http://www.dnspython.org/LICENSE -dogpile.cache!=0.9.1,!=1.1.7 # BSD +dnspython # http://www.dnspython.org/LICENSE +dogpile.cache!=1.1.7 # BSD dogtag-pki # LGPLv3+ -dulwich!=0.19.3,!=0.19.7 # Apache-2.0 +dulwich # Apache-2.0 edgegrid-python # Apache-2.0 elasticsearch<3.0.0 # Apache-2.0 enmerkar # BSD # NOTE: New versions of eventlet should not be accepted lightly # as they have earned a reputation of frequently breaking things. -eventlet!=0.18.3,!=0.20.1,!=0.21.0,!=0.23.0,!=0.25.0,!=0.32.0,!=0.34.1,!=0.34.2,!=0.34.3,!=0.35.0,!=0.36.0 # MIT -exabgp!=4.0.6 # BSD +eventlet!=0.34.1,!=0.34.2,!=0.34.3,!=0.35.0,!=0.36.0 # MIT +exabgp # BSD falcon # Apache-2.0 -# https://github.com/harlowja/fasteners/issues/36 -fasteners!=0.15,!=0.16 # Apache-2.0 -Flask!=0.11 # BSD +fasteners # Apache-2.0 +Flask # BSD Flask-RESTful # BSD GitPython # BSD License (3 clause) google-api-python-client # Apache-2.0 -graphviz!=0.5.0 # MIT License -greenlet!=0.4.14 # MIT +graphviz # MIT License +greenlet # MIT gunicorn # MIT httplib2 # MIT httpx # BSD @@ -64,7 +62,7 @@ icalendar # BSD # newer code than in [most] releases of the Python std library. importlib-metadata # Apache-2.0 infinisdk # BSD-3 -influxdb!=5.3.0 # MIT +influxdb # MIT influxdb-client # MIT infoblox-client # Apache-2.0 iso8601 # MIT @@ -72,33 +70,33 @@ jaeger-client # Apache-2.0 Jinja2 # BSD License (3 clause) jira # BSD License (2 clause) jmespath # MIT -jsonpatch!=1.20 # BSD +jsonpatch # BSD jsonschema # MIT kazoo # Apache-2.0 -kombu!=4.0.2 # BSD +kombu # BSD kubernetes # Apache-2.0 ldap3 # LGPLv3 -libvirt-python!=4.1.0,!=4.2.0 # LGPLv2+ -lxml!=3.7.0 # BSD +libvirt-python # LGPLv2+ +lxml # BSD Mako # MIT msgpack # Apache-2.0 munch # MIT ncclient # Apache-2.0 netaddr # BSD -netifaces!=0.10.0,!=0.10.1 # MIT +netifaces # MIT netmiko # MIT networkx!=2.8.4 # BSD oauthlib # BSD opentelemetry-exporter-otlp # Apache-2.0 opentelemetry-sdk # Apache-2.0 ovs # Apache-2.0 -packaging!=20.5,!=20.6,!=20.7 # Apache-2.0 -paramiko!=2.9.0,!=2.9.1 # LGPLv2.1+ +packaging # Apache-2.0 +paramiko # LGPLv2.1+ passlib # BSD Paste # MIT PasteDeploy # MIT -pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,!=1.4.0 # BSD -pexpect!=3.3 # ISC License +pecan # BSD +pexpect # ISC License pika # BSD Pillow # PIL License Pint # BSD @@ -107,27 +105,27 @@ PrettyTable!=3.4.0 # BSD prometheus-client # Apache-2.0 protobuf # BSD License (3 clause) psutil # BSD -pyasn1!=0.2.3 # BSD +pyasn1 # BSD pyasn1-lextudio # BSD pyasn1-modules # BSD pyasn1-modules-lextudio # BSD -pycadf!=2.0.0 # Apache-2.0 +pycadf # Apache-2.0 pycdlib # LGPLv2+ PyECLib # BSD -pyghmi!=1.4.0,!=1.5.11 # Apache-2.0 +pyghmi # Apache-2.0 PyJWT # MIT pykmip # Apache 2.0 License pylxd # Apache-2.0 -pymemcache!=1.3.0 # Apache 2.0 License +pymemcache # Apache 2.0 License PyMI;sys_platform=='win32' # Apache 2.0 License -pymongo!=3.1 # Apache-2.0 +pymongo # Apache-2.0 PyMySQL # MIT License pyngus # Apache-2.0 pyOpenSSL # Apache-2.0 pyparsing # MIT -pyroute2!=0.5.4,!=0.5.5,!=0.7.1;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2) -pysaml2!=4.0.3,!=4.0.4,!=4.0.5,!=4.0.5rc1,!=4.1.0,!=4.2.0,!=4.3.0,!=4.4.0,!=4.6.0 # Apache-2.0 -pyScss!=1.3.5 # MIT License +pyroute2!=0.7.1;sys_platform != "win32" # Apache-2.0 (+ dual licensed GPL2) +pysaml2 # Apache-2.0 +pyScss # MIT License pysnmp-lextudio # BSD pystache # MIT # Only required for sasl/binary protocol @@ -142,7 +140,7 @@ pywinrm # MIT PyYAML # MIT pyzabbix # LGPL rbd-iscsi-client # Apache-2.0 -requests!=2.20.0,!=2.24.0 # Apache-2.0 +requests # Apache-2.0 requests-aws # BSD License (3 clause) requests-kerberos # ISC requestsexceptions # Apache-2.0 @@ -150,18 +148,14 @@ rfc3986 # Apache-2.0 Routes # MIT rtslib-fb # Apache-2.0 ruamel.yaml # MIT -salt!=2019.2.1,!=2019.2.2 # Apache-2.0 +salt # Apache-2.0 scikit-learn # BSD scipy # BSD # https://github.com/holgern/py-scrypt/issues/16 scrypt!=0.8.21 # BSD semantic-version # BSD setproctitle # BSD -# NOTE(yamahata): -# bug work around of sqlalchemy -# https://bitbucket.org/zzzeek/sqlalchemy/issues/3952/ -# The fix which is in git master branch is planned for 1.1.9 -SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT +SQLAlchemy # MIT sqlalchemy-filters # Apache-2.0 sqlalchemy-migrate # Apache-2.0 SQLAlchemy-Utils # BSD License @@ -221,7 +215,7 @@ XStatic-Spin # MIT License XStatic-term.js # MIT License XStatic-tv4 # MIT # NOTE(anilvenkata): This is required for profiling oslo.service processes -Yappi!=0.98,!=0.99 # MIT +Yappi # MIT zeroconf # LGPL zipp # MIT zstd # BSD License (2 clause) @@ -230,7 +224,7 @@ zVMCloudConnector;sys_platform!='win32' # Apache 2.0 License ## section:testing bashate # Apache-2.0 -coverage!=4.4 # Apache-2.0 +coverage # Apache-2.0 ddt # MIT django-nose # BSD docker # Apache-2.0 @@ -245,7 +239,7 @@ ldappool # MPL # Do not make mock conditional on Python version: we depend on newer code than # in [most] releases of the Python std library. # https://github.com/testing-cabal/mock/issues/487 for 4.0.[0-1] blacklist -mock!=4.0.0,!=4.0.1 # BSD +mock # BSD moto # Apache-2.0 mypy # MIT nodeenv # BSD @@ -267,13 +261,13 @@ pytest-xdist # MIT python-consul # MIT License python-subunit # Apache-2.0/BSD pyzmq # LGPL+BSD -redis!=4.0.0 # MIT +redis # MIT requests-mock # Apache-2.0 -retrying!=1.3.0 # Apache-2.0 +retrying # Apache-2.0 sadisplay # BSD selenium<4.0.0 # Apache-2.0 -stestr!=2.3.0,!=3.0.0 # Apache-2.0 -sushy!=1.9.0 # Apache-2.0 +stestr # Apache-2.0 +sushy # Apache-2.0 tabulate # MIT tenacity # Apache-2.0 testrepository # Apache-2.0/BSD @@ -285,7 +279,7 @@ typing # PSF typing-extensions # PSF tzdata # MIT virtualbmc # Apache-2.0 -virtualenv!=16.3.0 # MIT +virtualenv # MIT WebTest # MIT Werkzeug!=2.2.0 # BSD License whereto # Apache-2.0 @@ -296,13 +290,13 @@ xvfbwrapper #license: MIT ## section:docs -blockdiag!=2.0.0 # Apache-2.0 +blockdiag # Apache-2.0 doc8 # Apache-2.0 pydot # MIT License pydotplus # MIT License Pygments # BSD license rst2txt # BSD -sphinx!=1.6.6,!=1.6.7,!=2.1.0,!=3.0.0,!=3.4.2 # BSD +sphinx # BSD sphinxcontrib-actdiag # BSD sphinxcontrib-apidoc # BSD sphinxcontrib-blockdiag # BSD @@ -327,16 +321,16 @@ ceilometer # Apache-2.0 # ceilometermiddleware might not show up with a search of setup.cfg and # requirements files, but some projects use it via being installed by devstack ceilometermiddleware # Apache-2.0 -cliff!=2.9.0,!=2.17.0 # Apache-2.0 +cliff # Apache-2.0 debtcollector # Apache-2.0 dib-utils # Apache-2.0 -diskimage-builder!=1.6.0,!=1.7.0,!=1.7.1 # Apache-2.0 -etcd3gw!=0.2.2,!=0.2.3,!=0.2.6 # Apache-2.0 +diskimage-builder # Apache-2.0 +etcd3gw # Apache-2.0 futurist # Apache-2.0 -glance-store!=0.29.0 # Apache-2.0 +glance-store # Apache-2.0 heat-translator # Apache-2.0 horizon # Apache-2.0 -ironic-lib!=4.6.0 # Apache-2.0 +ironic-lib # Apache-2.0 keystoneauth1 # Apache-2.0 keystonemiddleware # Apache-2.0 kuryr-lib # Apache-2.0 @@ -357,7 +351,7 @@ neutron-fwaas # Apache-2.0 neutron-lib # Apache-2.0 octavia-lib # Apache-2.0 os-apply-config # Apache-2.0 -os-brick!=2.8.0 # Apache-2.0 +os-brick # Apache-2.0 os-client-config # Apache-2.0 os-collect-config # Apache-2.0 os-ken # Apache-2.0 @@ -365,36 +359,34 @@ os-refresh-config # Apache-2.0 os-resource-classes # Apache-2.0 os-service-types # Apache-2.0 os-traits # Apache-2.0 -os-vif!=1.8.0,!=1.12.0,!=3.0.0 # Apache-2.0 +os-vif!=3.0.0 # Apache-2.0 os-win # Apache-2.0 osc-lib # Apache-2.0 osc-placement # Apache-2.0 -oslo.cache!=1.31.1,!=2.1.0 # Apache-2.0 +oslo.cache # Apache-2.0 oslo.concurrency # Apache-2.0 -oslo.config!=4.3.0,!=4.4.0 # Apache-2.0 +oslo.config # Apache-2.0 oslo.context # Apache-2.0 oslo.db # Apache-2.0 oslo.i18n # Apache-2.0 oslo.limit # Apache-2.0 -oslo.log!=3.44.2,!=4.1.2,!=4.2.0,!=5.0.1,!=5.0.2,!=5.1.0 # Apache-2.0 -oslo.messaging!=9.0.0 # Apache-2.0 +oslo.log!=5.0.1,!=5.0.2,!=5.1.0 # Apache-2.0 +oslo.messaging # Apache-2.0 oslo.metrics # Apache-2.0 oslo.middleware # Apache-2.0 -oslo.policy!=3.0.0,!=3.6.1 # Apache-2.0 +oslo.policy # Apache-2.0 oslo.privsep # Apache-2.0 oslo.reports # Apache-2.0 oslo.rootwrap # Apache-2.0 -# NOTE(mriedem): oslo.serialization 2.19.1 is blocked for bug 1593641 -oslo.serialization!=2.19.1 # Apache-2.0 -oslo.service!=1.28.1 # Apache-2.0 +oslo.serialization # Apache-2.0 +oslo.service # Apache-2.0 oslo.upgradecheck # Apache-2.0 -# NOTE(lajoskatona): oslo.utils version between 3.39.1 and 3.40.1 excluded due to bug 1812922 -oslo.utils!=3.39.1,!=3.40.0,!=3.40.1 # Apache-2.0 +oslo.utils # Apache-2.0 oslo.versionedobjects # Apache-2.0 oslo.vmware # Apache-2.0 osprofiler # Apache-2.0 -pbr!=2.1.0 # Apache-2.0 -stevedore!=3.0.0 # Apache-2.0 +pbr # Apache-2.0 +stevedore # Apache-2.0 tap-as-a-service # Apache-2.0 taskflow # Apache-2.0 tempest # Apache-2.0 @@ -413,7 +405,7 @@ gnocchiclient # Apache-2.0 openstacksdk # Apache-2.0 python-barbicanclient # Apache-2.0 python-blazarclient # Apache-2.0 -python-cinderclient!=4.0.0 # Apache-2.0 +python-cinderclient # Apache-2.0 python-cloudkittyclient # Apache-2.0 python-cyborgclient # Apache-2.0 python-designateclient # Apache-2.0 @@ -421,12 +413,12 @@ python-freezerclient # Apache-2.0 python-glanceclient # Apache-2.0 python-heatclient # Apache-2.0 python-ironic-inspector-client # Apache-2.0 -python-ironicclient!=2.5.2,!=2.7.1,!=3.0.0 # Apache-2.0 -python-keystoneclient!=2.1.0 # Apache-2.0 +python-ironicclient # Apache-2.0 +python-keystoneclient # Apache-2.0 python-magnumclient # Apache-2.0 python-manilaclient # Apache-2.0 python-masakariclient # Apache-2.0 -python-mistralclient!=3.2.0 # Apache-2.0 +python-mistralclient # Apache-2.0 python-monascaclient # Apache-2.0 python-muranoclient # Apache-2.0 python-neutronclient # Apache-2.0 @@ -449,7 +441,7 @@ python-zunclient # Apache-2.0 ## ## Docs-related projects under openstack governance -openstackdocstheme!=2.1.0,!=2.1.1 # Apache-2.0 +openstackdocstheme # Apache-2.0 os-api-ref # Apache-2.0 oslosphinx # Apache-2.0 reno # Apache-2.0 @@ -479,7 +471,7 @@ python-linstor # LGPLv3 pywbem # LGPLv2.1+ rsd-lib # Apache-2.0 storops # Apache-2.0 -storpool!=5.2.0,!=5.3.0 # Apache-2.0 +storpool # Apache-2.0 storpool.spopenstack # Apache-2.0 ## section:internal @@ -506,7 +498,7 @@ extras # MIT jsonpath-rw # Apache-2.0 jsonpath-rw-ext # Apache-2.0 kafka-python # Apache-2.0 -oauth2client!=4.0.0 # Apache-2.0 +oauth2client # Apache-2.0 pyinotify;sys_platform!='win32' and sys_platform!='darwin' and sys_platform!='sunos5' # MIT # pysnmp library is not maintained since 4 years, it is # not recommended to use it, use its fork pysnmp-lextudio instead diff --git a/tools/lint.py b/tools/lint.py index 7ac319bf56..e30edfde31 100755 --- a/tools/lint.py +++ b/tools/lint.py @@ -12,13 +12,20 @@ # License for the specific language governing permissions and limitations # under the License. +from concurrent import futures +import datetime import os +import sys + +from packaging import requirements +import requests GLOBAL_REQS = os.path.join( os.path.dirname(os.path.realpath(__file__)), '..', 'global-requirements.txt', ) +MAX_EXCLUDE_AGE = datetime.timedelta(365 * 2) def sort() -> None: @@ -75,5 +82,120 @@ def sort() -> None: fh.write(dep) +def validate_excludes( + name: str, specifiers: requirements.SpecifierSet +) -> tuple[str, str]: + data = requests.get(f'https://pypi.org/pypi/{name}/json').json() + latest_release = max(data['releases']) + + result = [] + for specifier in specifiers: + if specifier.operator != '!=': + result.append(specifier) + # non-exclusion specifier + continue + + exclude = specifier.version + + if exclude == latest_release: + print( + f'Release {exclude} is the latest release for package {name}. ' + f'Skipping checks.' + ) + result.append(specifier) + continue + + release = data['releases'].get(exclude) + if not release: + print( + f'Failed to find release {exclude} for package {name}', + file=sys.stderr, + ) + continue + + if all(r['yanked'] for r in release): + print(f'Release {exclude} for package {name} was yanked') + continue + + now = datetime.datetime.now(datetime.timezone.utc) + age = min( + (now - datetime.datetime.fromisoformat(r['upload_time_iso_8601'])) + for r in release + ) + if age >= MAX_EXCLUDE_AGE: + print( + f'Release {exclude} for package {name} is older than the ' + f'upper limit for age ' + f'({age.days} days >= {MAX_EXCLUDE_AGE.days} days)' + ) + continue + + # exclude is recent enough and not yanked so keep it + result.append(specifier) + + return name, ','.join(sorted(str(r) for r in result)) + + +def remove_old_excludes(): + """Remove excludes for old package versions. + + If we exclude e.g. v1.22 of a package but that version was release over 2 + years ago and said package is currently at v1.45, then there's no reason to + keep that exclude around. + """ + deps: dict[str, set[str]] = {} + + with open(GLOBAL_REQS) as fh: + for line in fh.readlines(): + if not line.strip() or line.startswith('#'): + # ignore blank lines and comments + continue + + req = requirements.Requirement(line.split(' #')[0]) + + if req.name in ('setuptools',): + # ignore certain packages where we want to retain all excludes + continue + + # these shouldn't be in our global-requirements file so we don't + # handle them...but make sure + assert not req.extras, f'unexpected extras: {req}' + assert not req.url, f'unexpected url: {req}' + + if any(s.operator == '!=' for s in req.specifier): + deps[req.name] = req.specifier + + with futures.ThreadPoolExecutor() as executor: + res = executor.map(validate_excludes, *zip(*deps.items())) + + deps.update(dict(res)) + + with open(GLOBAL_REQS) as fh: + data = fh.read() + + with open(GLOBAL_REQS, 'w') as fh: + for i, line in enumerate(data.split('\n')): + if i != 0: + fh.write('\n') + + if line.startswith('#') or not line.strip(): + # skipped (empty or comment) + fh.write(line) + continue + + dep, comment, license = line.partition(' #') + req = requirements.Requirement(dep) + if req.name not in deps: + # skipped (no cap) + fh.write(line) + continue + + req.specifier = deps[req.name] + # requirements.Requirement.__str__ adds a space after the semicolon + # which we don't want + fh.write(str(req).replace('; ', ';') + comment + license) + + if __name__ == '__main__': + remove_old_excludes() sort()