Merge "Add ability to strip old excludes"

global-requirements.txt

@@ -1,14 +1,14 @@
 ## section:general
 
-alembic!=1.2.0,!=1.6.3  # MIT
+alembic  # MIT
-amqp!=2.1.4,!=5.0.4  # BSD
+amqp  # BSD
-ansible-runner!=1.3.5  # Apache 2.0
+ansible-runner  # Apache 2.0
 appdirs  # MIT License
 apscheduler  # MIT License
 autobahn  # MIT License
 automaton  # Apache-2.0
 autopage  # Apache-2.0
-Babel!=2.4.0  # BSD
+Babel  # BSD
 bcrypt  # Apache-2.0
 beautifulsoup4  # MIT
 betamax  # Apache-2.0
@@ -16,15 +16,15 @@ boto  # MIT
 boto3  # Apache-2.0
 botocore  # Apache-2.0
 cachetools  # MIT License
-cassandra-driver!=3.6.0  # Apache-2.0
+cassandra-driver  # Apache-2.0
 cffi  # MIT
-cmd2!=0.8.3  # MIT
+cmd2  # MIT
-confluent-kafka!=1.4.0  # Apache-2.0
+confluent-kafka  # Apache-2.0
 confspirator  # Apache-2.0
 construct  # MIT
 cotyledon  # Apache-2.0
 croniter  # MIT License
-cryptography!=2.0  # BSD/Apache-2.0
+cryptography  # BSD/Apache-2.0
 cursive  # Apache-2.0
 datetimerange  # MIT
 decorator  # BSD
@@ -34,27 +34,25 @@ django-debreach  # BSD
 django-formtools  # BSD
 django-pyscss  # BSD License (2 clause)
 Django<4.3  # BSD
-# eventlet is not compatibile with 2.0.0: https://github.com/eventlet/eventlet/issues/619
+dnspython  # http://www.dnspython.org/LICENSE
-dnspython!=2.0.0,!=2.2.0  # http://www.dnspython.org/LICENSE
+dogpile.cache!=1.1.7  # BSD
-dogpile.cache!=0.9.1,!=1.1.7  # BSD
 dogtag-pki  # LGPLv3+
-dulwich!=0.19.3,!=0.19.7  # Apache-2.0
+dulwich  # Apache-2.0
 edgegrid-python  # Apache-2.0
 elasticsearch<3.0.0  # Apache-2.0
 enmerkar  # BSD
 # NOTE: New versions of eventlet should not be accepted lightly
 # as they have earned a reputation of frequently breaking things.
-eventlet!=0.18.3,!=0.20.1,!=0.21.0,!=0.23.0,!=0.25.0,!=0.32.0,!=0.34.1,!=0.34.2,!=0.34.3,!=0.35.0,!=0.36.0  # MIT
+eventlet!=0.34.1,!=0.34.2,!=0.34.3,!=0.35.0,!=0.36.0  # MIT
-exabgp!=4.0.6  # BSD
+exabgp  # BSD
 falcon  # Apache-2.0
-# https://github.com/harlowja/fasteners/issues/36
+fasteners  # Apache-2.0
-fasteners!=0.15,!=0.16  # Apache-2.0
+Flask  # BSD
-Flask!=0.11  # BSD
 Flask-RESTful  # BSD
 GitPython  # BSD License (3 clause)
 google-api-python-client  # Apache-2.0
-graphviz!=0.5.0  # MIT License
+graphviz  # MIT License
-greenlet!=0.4.14  # MIT
+greenlet  # MIT
 gunicorn  # MIT
 httplib2  # MIT
 httpx  # BSD
@@ -64,7 +62,7 @@ icalendar  # BSD
 # newer code than in [most] releases of the Python std library.
 importlib-metadata  # Apache-2.0
 infinisdk  # BSD-3
-influxdb!=5.3.0  # MIT
+influxdb  # MIT
 influxdb-client  # MIT
 infoblox-client  # Apache-2.0
 iso8601  # MIT
@@ -72,33 +70,33 @@ jaeger-client  # Apache-2.0
 Jinja2  # BSD License (3 clause)
 jira  # BSD License (2 clause)
 jmespath  # MIT
-jsonpatch!=1.20  # BSD
+jsonpatch  # BSD
 jsonschema  # MIT
 kazoo  # Apache-2.0
-kombu!=4.0.2  # BSD
+kombu  # BSD
 kubernetes  # Apache-2.0
 ldap3  # LGPLv3
-libvirt-python!=4.1.0,!=4.2.0  # LGPLv2+
+libvirt-python  # LGPLv2+
-lxml!=3.7.0  # BSD
+lxml  # BSD
 Mako  # MIT
 msgpack  # Apache-2.0
 munch  # MIT
 ncclient  # Apache-2.0
 netaddr  # BSD
-netifaces!=0.10.0,!=0.10.1  # MIT
+netifaces  # MIT
 netmiko  # MIT
 networkx!=2.8.4  # BSD
 oauthlib  # BSD
 opentelemetry-exporter-otlp  # Apache-2.0
 opentelemetry-sdk  # Apache-2.0
 ovs  # Apache-2.0
-packaging!=20.5,!=20.6,!=20.7  # Apache-2.0
+packaging  # Apache-2.0
-paramiko!=2.9.0,!=2.9.1  # LGPLv2.1+
+paramiko  # LGPLv2.1+
 passlib  # BSD
 Paste  # MIT
 PasteDeploy  # MIT
-pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,!=1.4.0  # BSD
+pecan  # BSD
-pexpect!=3.3  # ISC License
+pexpect  # ISC License
 pika  # BSD
 Pillow  # PIL License
 Pint  # BSD
@@ -107,27 +105,27 @@ PrettyTable!=3.4.0  # BSD
 prometheus-client  # Apache-2.0
 protobuf  # BSD License (3 clause)
 psutil  # BSD
-pyasn1!=0.2.3  # BSD
+pyasn1  # BSD
 pyasn1-lextudio  # BSD
 pyasn1-modules  # BSD
 pyasn1-modules-lextudio  # BSD
-pycadf!=2.0.0  # Apache-2.0
+pycadf  # Apache-2.0
 pycdlib  # LGPLv2+
 PyECLib  # BSD
-pyghmi!=1.4.0,!=1.5.11  # Apache-2.0
+pyghmi  # Apache-2.0
 PyJWT  # MIT
 pykmip  # Apache 2.0 License
 pylxd  # Apache-2.0
-pymemcache!=1.3.0  # Apache 2.0 License
+pymemcache  # Apache 2.0 License
 PyMI;sys_platform=='win32'  # Apache 2.0 License
-pymongo!=3.1  # Apache-2.0
+pymongo  # Apache-2.0
 PyMySQL  # MIT License
 pyngus  # Apache-2.0
 pyOpenSSL  # Apache-2.0
 pyparsing  # MIT
-pyroute2!=0.5.4,!=0.5.5,!=0.7.1;sys_platform!='win32'  # Apache-2.0 (+ dual licensed GPL2)
+pyroute2!=0.7.1;sys_platform != "win32"  # Apache-2.0 (+ dual licensed GPL2)
-pysaml2!=4.0.3,!=4.0.4,!=4.0.5,!=4.0.5rc1,!=4.1.0,!=4.2.0,!=4.3.0,!=4.4.0,!=4.6.0  # Apache-2.0
+pysaml2  # Apache-2.0
-pyScss!=1.3.5  # MIT License
+pyScss  # MIT License
 pysnmp-lextudio  # BSD
 pystache  # MIT
 # Only required for sasl/binary protocol
@@ -142,7 +140,7 @@ pywinrm  # MIT
 PyYAML  # MIT
 pyzabbix  # LGPL
 rbd-iscsi-client  # Apache-2.0
-requests!=2.20.0,!=2.24.0  # Apache-2.0
+requests  # Apache-2.0
 requests-aws  # BSD License (3 clause)
 requests-kerberos  # ISC
 requestsexceptions  # Apache-2.0
@@ -150,18 +148,14 @@ rfc3986  # Apache-2.0
 Routes  # MIT
 rtslib-fb  # Apache-2.0
 ruamel.yaml  # MIT
-salt!=2019.2.1,!=2019.2.2  # Apache-2.0
+salt  # Apache-2.0
 scikit-learn  # BSD
 scipy  # BSD
 # https://github.com/holgern/py-scrypt/issues/16
 scrypt!=0.8.21  # BSD
 semantic-version  # BSD
 setproctitle  # BSD
-# NOTE(yamahata):
+SQLAlchemy  # MIT
-# bug work around of sqlalchemy
-# https://bitbucket.org/zzzeek/sqlalchemy/issues/3952/
-# The fix which is in git master branch is planned for 1.1.9
-SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8  # MIT
 sqlalchemy-filters  # Apache-2.0
 sqlalchemy-migrate  # Apache-2.0
 SQLAlchemy-Utils  # BSD License
@@ -221,7 +215,7 @@ XStatic-Spin  # MIT License
 XStatic-term.js  # MIT License
 XStatic-tv4  # MIT
 # NOTE(anilvenkata): This is required for profiling oslo.service processes
-Yappi!=0.98,!=0.99  # MIT
+Yappi  # MIT
 zeroconf  # LGPL
 zipp  # MIT
 zstd  # BSD License (2 clause)
@@ -230,7 +224,7 @@ zVMCloudConnector;sys_platform!='win32'  # Apache 2.0 License
 ## section:testing
 
 bashate  # Apache-2.0
-coverage!=4.4  # Apache-2.0
+coverage  # Apache-2.0
 ddt  # MIT
 django-nose  # BSD
 docker  # Apache-2.0
@@ -245,7 +239,7 @@ ldappool  # MPL
 # Do not make mock conditional on Python version: we depend on newer code than
 # in [most] releases of the Python std library.
 # https://github.com/testing-cabal/mock/issues/487  for 4.0.[0-1] blacklist
-mock!=4.0.0,!=4.0.1  # BSD
+mock  # BSD
 moto  # Apache-2.0
 mypy  # MIT
 nodeenv  # BSD
@@ -267,13 +261,13 @@ pytest-xdist  # MIT
 python-consul  # MIT License
 python-subunit  # Apache-2.0/BSD
 pyzmq  # LGPL+BSD
-redis!=4.0.0  # MIT
+redis  # MIT
 requests-mock  # Apache-2.0
-retrying!=1.3.0  # Apache-2.0
+retrying  # Apache-2.0
 sadisplay  # BSD
 selenium<4.0.0  # Apache-2.0
-stestr!=2.3.0,!=3.0.0  # Apache-2.0
+stestr  # Apache-2.0
-sushy!=1.9.0  # Apache-2.0
+sushy  # Apache-2.0
 tabulate  # MIT
 tenacity  # Apache-2.0
 testrepository  # Apache-2.0/BSD
@@ -289,7 +283,7 @@ typing  # PSF
 typing-extensions  # PSF
 tzdata  # MIT
 virtualbmc  # Apache-2.0
-virtualenv!=16.3.0  # MIT
+virtualenv  # MIT
 WebTest  # MIT
 Werkzeug!=2.2.0  # BSD License
 whereto  # Apache-2.0
@@ -300,13 +294,13 @@ xvfbwrapper  #license: MIT
 
 ## section:docs
 
-blockdiag!=2.0.0  # Apache-2.0
+blockdiag  # Apache-2.0
 doc8  # Apache-2.0
 pydot  # MIT License
 pydotplus  # MIT License
 Pygments  # BSD license
 rst2txt  # BSD
-sphinx!=1.6.6,!=1.6.7,!=2.1.0,!=3.0.0,!=3.4.2  # BSD
+sphinx  # BSD
 sphinxcontrib-actdiag  # BSD
 sphinxcontrib-apidoc  # BSD
 sphinxcontrib-blockdiag  # BSD
@@ -331,16 +325,16 @@ ceilometer  # Apache-2.0
 # ceilometermiddleware might not show up with a search of setup.cfg and
 # requirements files, but some projects use it via being installed by devstack
 ceilometermiddleware  # Apache-2.0
-cliff!=2.9.0,!=2.17.0  # Apache-2.0
+cliff  # Apache-2.0
 debtcollector  # Apache-2.0
 dib-utils  # Apache-2.0
-diskimage-builder!=1.6.0,!=1.7.0,!=1.7.1  # Apache-2.0
+diskimage-builder  # Apache-2.0
-etcd3gw!=0.2.2,!=0.2.3,!=0.2.6  # Apache-2.0
+etcd3gw  # Apache-2.0
 futurist  # Apache-2.0
-glance-store!=0.29.0  # Apache-2.0
+glance-store  # Apache-2.0
 heat-translator  # Apache-2.0
 horizon  # Apache-2.0
-ironic-lib!=4.6.0  # Apache-2.0
+ironic-lib  # Apache-2.0
 keystoneauth1  # Apache-2.0
 keystonemiddleware  # Apache-2.0
 kuryr-lib  # Apache-2.0
@@ -361,7 +355,7 @@ neutron-fwaas  # Apache-2.0
 neutron-lib  # Apache-2.0
 octavia-lib  # Apache-2.0
 os-apply-config  # Apache-2.0
-os-brick!=2.8.0  # Apache-2.0
+os-brick  # Apache-2.0
 os-client-config  # Apache-2.0
 os-collect-config  # Apache-2.0
 os-ken  # Apache-2.0
@@ -369,36 +363,34 @@ os-refresh-config  # Apache-2.0
 os-resource-classes  # Apache-2.0
 os-service-types  # Apache-2.0
 os-traits  # Apache-2.0
-os-vif!=1.8.0,!=1.12.0,!=3.0.0  # Apache-2.0
+os-vif!=3.0.0  # Apache-2.0
 os-win  # Apache-2.0
 osc-lib  # Apache-2.0
 osc-placement  # Apache-2.0
-oslo.cache!=1.31.1,!=2.1.0  # Apache-2.0
+oslo.cache  # Apache-2.0
 oslo.concurrency  # Apache-2.0
-oslo.config!=4.3.0,!=4.4.0  # Apache-2.0
+oslo.config  # Apache-2.0
 oslo.context  # Apache-2.0
 oslo.db  # Apache-2.0
 oslo.i18n  # Apache-2.0
 oslo.limit  # Apache-2.0
-oslo.log!=3.44.2,!=4.1.2,!=4.2.0,!=5.0.1,!=5.0.2,!=5.1.0  # Apache-2.0
+oslo.log!=5.0.1,!=5.0.2,!=5.1.0  # Apache-2.0
-oslo.messaging!=9.0.0  # Apache-2.0
+oslo.messaging  # Apache-2.0
 oslo.metrics  # Apache-2.0
 oslo.middleware  # Apache-2.0
-oslo.policy!=3.0.0,!=3.6.1  # Apache-2.0
+oslo.policy  # Apache-2.0
 oslo.privsep  # Apache-2.0
 oslo.reports  # Apache-2.0
 oslo.rootwrap  # Apache-2.0
-# NOTE(mriedem): oslo.serialization 2.19.1 is blocked for bug 1593641
+oslo.serialization  # Apache-2.0
-oslo.serialization!=2.19.1  # Apache-2.0
+oslo.service  # Apache-2.0
-oslo.service!=1.28.1  # Apache-2.0
 oslo.upgradecheck  # Apache-2.0
-# NOTE(lajoskatona): oslo.utils version between 3.39.1 and 3.40.1 excluded due to bug 1812922
+oslo.utils  # Apache-2.0
-oslo.utils!=3.39.1,!=3.40.0,!=3.40.1  # Apache-2.0
 oslo.versionedobjects  # Apache-2.0
 oslo.vmware  # Apache-2.0
 osprofiler  # Apache-2.0
-pbr!=2.1.0  # Apache-2.0
+pbr  # Apache-2.0
-stevedore!=3.0.0  # Apache-2.0
+stevedore  # Apache-2.0
 tap-as-a-service  # Apache-2.0
 taskflow  # Apache-2.0
 tempest  # Apache-2.0
@@ -417,7 +409,7 @@ gnocchiclient  # Apache-2.0
 openstacksdk  # Apache-2.0
 python-barbicanclient  # Apache-2.0
 python-blazarclient  # Apache-2.0
-python-cinderclient!=4.0.0  # Apache-2.0
+python-cinderclient  # Apache-2.0
 python-cloudkittyclient  # Apache-2.0
 python-cyborgclient  # Apache-2.0
 python-designateclient  # Apache-2.0
@@ -425,12 +417,12 @@ python-freezerclient  # Apache-2.0
 python-glanceclient  # Apache-2.0
 python-heatclient  # Apache-2.0
 python-ironic-inspector-client  # Apache-2.0
-python-ironicclient!=2.5.2,!=2.7.1,!=3.0.0  # Apache-2.0
+python-ironicclient  # Apache-2.0
-python-keystoneclient!=2.1.0  # Apache-2.0
+python-keystoneclient  # Apache-2.0
 python-magnumclient  # Apache-2.0
 python-manilaclient  # Apache-2.0
 python-masakariclient  # Apache-2.0
-python-mistralclient!=3.2.0  # Apache-2.0
+python-mistralclient  # Apache-2.0
 python-monascaclient  # Apache-2.0
 python-muranoclient  # Apache-2.0
 python-neutronclient  # Apache-2.0
@@ -453,7 +445,7 @@ python-zunclient  # Apache-2.0
 ##
 ## Docs-related projects under openstack governance
 
-openstackdocstheme!=2.1.0,!=2.1.1  # Apache-2.0
+openstackdocstheme  # Apache-2.0
 os-api-ref  # Apache-2.0
 oslosphinx  # Apache-2.0
 reno  # Apache-2.0
@@ -483,7 +475,7 @@ python-linstor  # LGPLv3
 pywbem  # LGPLv2.1+
 rsd-lib  # Apache-2.0
 storops  # Apache-2.0
-storpool!=5.2.0,!=5.3.0  # Apache-2.0
+storpool  # Apache-2.0
 storpool.spopenstack  # Apache-2.0
 
 ## section:internal
@@ -510,7 +502,7 @@ extras  # MIT
 jsonpath-rw  # Apache-2.0
 jsonpath-rw-ext  # Apache-2.0
 kafka-python  # Apache-2.0
-oauth2client!=4.0.0  # Apache-2.0
+oauth2client  # Apache-2.0
 pyinotify;sys_platform!='win32' and sys_platform!='darwin' and sys_platform!='sunos5'  # MIT
 # pysnmp library is not maintained since 4 years, it is
 # not recommended to use it, use its fork pysnmp-lextudio instead

tools/lint.py

@@ -12,13 +12,20 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from concurrent import futures
+import datetime
 import os
+import sys
+
+from packaging import requirements
+import requests
 
 GLOBAL_REQS = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
     '..',
     'global-requirements.txt',
 )
+MAX_EXCLUDE_AGE = datetime.timedelta(365 * 2)
 
 
 def sort() -> None:
@@ -75,5 +82,120 @@ def sort() -> None:
                 fh.write(dep)
 
 
+def validate_excludes(
+    name: str, specifiers: requirements.SpecifierSet
+) -> tuple[str, str]:
+    data = requests.get(f'https://pypi.org/pypi/{name}/json').json()
+    latest_release = max(data['releases'])
+
+    result = []
+    for specifier in specifiers:
+        if specifier.operator != '!=':
+            result.append(specifier)
+            # non-exclusion specifier
+            continue
+
+        exclude = specifier.version
+
+        if exclude == latest_release:
+            print(
+                f'Release {exclude} is the latest release for package {name}. '
+                f'Skipping checks.'
+            )
+            result.append(specifier)
+            continue
+
+        release = data['releases'].get(exclude)
+        if not release:
+            print(
+                f'Failed to find release {exclude} for package {name}',
+                file=sys.stderr,
+            )
+            continue
+
+        if all(r['yanked'] for r in release):
+            print(f'Release {exclude} for package {name} was yanked')
+            continue
+
+        now = datetime.datetime.now(datetime.timezone.utc)
+        age = min(
+            (now - datetime.datetime.fromisoformat(r['upload_time_iso_8601']))
+            for r in release
+        )
+        if age >= MAX_EXCLUDE_AGE:
+            print(
+                f'Release {exclude} for package {name} is older than the '
+                f'upper limit for age '
+                f'({age.days} days >= {MAX_EXCLUDE_AGE.days} days)'
+            )
+            continue
+
+        # exclude is recent enough and not yanked so keep it
+        result.append(specifier)
+
+    return name, ','.join(sorted(str(r) for r in result))
+
+
+def remove_old_excludes():
+    """Remove excludes for old package versions.
+
+    If we exclude e.g. v1.22 of a package but that version was release over 2
+    years ago and said package is currently at v1.45, then there's no reason to
+    keep that exclude around.
+    """
+    deps: dict[str, set[str]] = {}
+
+    with open(GLOBAL_REQS) as fh:
+        for line in fh.readlines():
+            if not line.strip() or line.startswith('#'):
+                # ignore blank lines and comments
+                continue
+
+            req = requirements.Requirement(line.split(' #')[0])
+
+            if req.name in ('setuptools',):
+                # ignore certain packages where we want to retain all excludes
+                continue
+
+            # these shouldn't be in our global-requirements file so we don't
+            # handle them...but make sure
+            assert not req.extras, f'unexpected extras: {req}'
+            assert not req.url, f'unexpected url: {req}'
+
+            if any(s.operator == '!=' for s in req.specifier):
+                deps[req.name] = req.specifier
+
+    with futures.ThreadPoolExecutor() as executor:
+        res = executor.map(validate_excludes, *zip(*deps.items()))
+
+    deps.update(dict(res))
+
+    with open(GLOBAL_REQS) as fh:
+        data = fh.read()
+
+    with open(GLOBAL_REQS, 'w') as fh:
+        for i, line in enumerate(data.split('\n')):
+            if i != 0:
+                fh.write('\n')
+
+            if line.startswith('#') or not line.strip():
+                # skipped (empty or comment)
+                fh.write(line)
+                continue
+
+            dep, comment, license = line.partition('  #')
+            req = requirements.Requirement(dep)
+            if req.name not in deps:
+                # skipped (no cap)
+                fh.write(line)
+                continue
+
+            req.specifier = deps[req.name]
+            # requirements.Requirement.__str__ adds a space after the semicolon
+            # which we don't want
+            fh.write(str(req).replace('; ', ';') + comment + license)
+
+
 if __name__ == '__main__':
+    remove_old_excludes()
     sort()