diff --git a/devstack/settings b/devstack/settings
index 8cab773d..df3c8186 100644
--- a/devstack/settings
+++ b/devstack/settings
@@ -24,6 +24,7 @@ KURYR_PROCESS_EXTERNAL_CONNECTIVITY=${KURYR_PROCESS_EXTERNAL_CONNECTIVITY:-True}
 
 KURYR_DOCKER_ENGINE_PORT=${KURYR_DOCKER_ENGINE_PORT:-2375}
 DOCKER_CLUSTER_STORE=${DOCKER_CLUSTER_STORE:-etcd://$SERVICE_HOST:$ETCD_PORT}
+UBUNTU_DOCKER_VERSION=${UBUNTU_DOCKER_VERSION:-5:20.10.23~3-0~ubuntu-jammy}
 
 GITREPO["kuryr"]=${KURYR_REPO:-${GIT_BASE}/openstack/kuryr.git}
 GITBRANCH["kuryr"]=${KURYR_BRANCH:-$TARGET_BRANCH}
diff --git a/kuryr_libnetwork/controllers.py b/kuryr_libnetwork/controllers.py
index dfb0c62e..288f3688 100644
--- a/kuryr_libnetwork/controllers.py
+++ b/kuryr_libnetwork/controllers.py
@@ -1859,6 +1859,21 @@ def ipam_release_pool():
                       'subnetpool.')
             return flask.jsonify(const.SCHEMA['SUCCESS'])
 
+    # Delete subnets created by kuryr
+    filtered_subnets = _get_subnets_by_attrs(
+        subnetpool_id=pool_id)
+    for subnet in filtered_subnets:
+        try:
+            subnet_name = subnet.get('name')
+            if str(subnet_name).startswith(const.SUBNET_NAME_PREFIX):
+                app.neutron.delete_subnet(subnet['id'])
+        except n_exceptions.Conflict:
+            LOG.error("Subnet %s is in use, "
+                      "can't be deleted.", subnet['id'])
+        except n_exceptions.NeutronClientException as ex:
+            LOG.error("Error happened during deleting a "
+                      "subnet created by kuryr: %s", ex)
+
     try:
         app.neutron.delete_subnetpool(pool_id)
     except n_exceptions.Conflict:
diff --git a/kuryr_libnetwork/tests/unit/test_kuryr_ipam.py b/kuryr_libnetwork/tests/unit/test_kuryr_ipam.py
index 3ea5c317..dbc89147 100644
--- a/kuryr_libnetwork/tests/unit/test_kuryr_ipam.py
+++ b/kuryr_libnetwork/tests/unit/test_kuryr_ipam.py
@@ -570,7 +570,7 @@ class TestKuryrIpam(base.TestKuryrBase):
         if mock_app.tag_ext:
             mock_list_subnetpools.assert_called_with(
                 id=fake_kuryr_subnetpool_id)
-            mock_list_subnets.assert_called_with(
+            mock_list_subnets.assert_any_call(
                 cidr=FAKE_IP4_CIDR)
             mock_remove_tag.assert_called_with('subnets',
                                                subnet_v4_id,
diff --git a/tox.ini b/tox.ini
index 9005d194..24e0fc4b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,10 +1,8 @@
 [tox]
 minversion = 2.3.1
 envlist = py38,pep8
-skipsdist = True
 
 [testenv]
-basepython = python3
 # Note the hash seed is set to 0 until neutron can be tested with a
 # random hash seed successfully.
 setenv = VIRTUAL_ENV={envdir}
@@ -16,7 +14,7 @@ deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/up
        -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 
-whitelist_externals = sh
+allowlist_externals = sh
                       find
 commands = find . -type f -name "*.py[c|o]" -delete
            stestr run {posargs}