Switch to 2023.1 Python3 unit tests and generic template name

This is an automatically generated patch to ensure unit testing is in place for all the of the tested runtimes for antelope. Also, updating the template name to generic one. See also the PTI in governance [1]. [1]: https://governance.openstack.org/tc/reference/project-testing-interface.html Change-Id: I29fcb9afa016b1321b0b4c0118a8d7ffbc00b02e
2022-09-15 15:59:06 +00:00 · 2022-09-15 15:59:06 +00:00 · 8049697fc9
parent 2fbc7937bb
commit 8049697fc9
9 changed files with 90 additions and 54 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -75,8 +75,8 @@

 - project:
    templates:
-      - openstack-python3-zed-jobs
-      - openstack-python3-zed-jobs-arm64
+      - openstack-python3-jobs
+      - openstack-python3-jobs-arm64
      - publish-openstack-docs-pti
      - periodic-stable-jobs
      - check-requirements
@ -92,16 +92,12 @@
        - zaqar-grenade-mongodb
        - zaqar-grenade-redis
        - zaqar-grenade-swift
-        # Heat job that deploy Zaqar
-        - heat-functional-convg-mysql-lbaasv2
        - zaqar-rally:
            voting: false
    gate:
-      queue: zaqar
      jobs:
        - zaqar-tox-integration
        - zaqar-tempest-plugin-mongodb
        - zaqar-tempest-plugin-redis
        - zaqar-tempest-plugin-swift
        - zaqar-tempest-plugin-swift-ipv6
-        - heat-functional-convg-mysql-lbaasv2
--- a/bindep.txt
+++ b/bindep.txt
@ -3,7 +3,6 @@
 # their own bindep.txt files if the list below is not
 # working for them.

-mongodb [platform:dpkg]
 mongodb-server [platform:rpm !platform:opensuseproject]
 dev-db/mongodb [platform:gentoo]
 redis [platform:rpm]
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@ -50,15 +50,16 @@ function cleanup_zaqar {

 # cleanup_zaqar_mongodb() - Remove residual data files, anything left over from previous
 # runs that a clean run would need to clean up
+# After mongodb 6.0, the mongo shell has been remove, now using mongosh.
 function cleanup_zaqar_mongodb {
-    if ! timeout $SERVICE_TIMEOUT sh -c "while ! mongo zaqar --eval 'db.dropDatabase();'; do sleep 1; done"; then
+    if ! timeout $SERVICE_TIMEOUT sh -c "while ! mongosh zaqar --eval 'db.dropDatabase();'; do sleep 1; done"; then
        die $LINENO "Mongo DB did not start"
    else
-        full_version=$(mongo zaqar --eval 'db.dropDatabase();')
-        mongo_version=`echo $full_version | cut -d' ' -f4`
-        required_mongo_version='2.2'
+        full_version=$(mongosh zaqar --eval 'db.dropDatabase();')
+        mongo_version=`echo $full_version | cut -d' ' -f11`
+        required_mongo_version='6.0'
        if [[ $mongo_version < $required_mongo_version ]]; then
-            die $LINENO "Zaqar needs Mongo DB version >= 2.2 to run."
+            die $LINENO "Zaqar needs Mongo DB version >= 6.0 to run."
        fi
    fi
 }
@ -186,15 +187,18 @@ function configure_mongodb {
    # per database.
    pip_install pymongo
    if is_ubuntu; then
-        install_package mongodb-server
-        if ! grep -qF "smallfiles = true" /etc/mongodb.conf; then
-            echo "smallfiles = true" | sudo tee --append /etc/mongodb.conf > /dev/null
-        fi
-        restart_service mongodb
+        # NOTE: To fix the mongodb's issue in ubuntu 22.04 LTS
+        wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add -
+        echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list
+        sudo apt update
+        curl -LO http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
+        sudo dpkg -i ./libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
+
+        install_package mongodb-org
+        restart_service mongod
    elif is_fedora; then
        install_package mongodb
        install_package mongodb-server
-        sudo sed -i '/--smallfiles/!s/OPTIONS=\"/OPTIONS=\"--smallfiles /' /etc/sysconfig/mongod
        restart_service mongod
    fi
 }
--- a/requirements.txt
+++ b/requirements.txt
@ -16,17 +16,17 @@ python-swiftclient>=3.10.1 # Apache-2.0
 WebOb>=1.7.1 # MIT
 stevedore>=3.2.2 # Apache-2.0
 oslo.cache>=1.26.0 # Apache-2.0
-oslo.config>=6.8.0 # Apache-2.0
+oslo.config>=8.3.2 # Apache-2.0
 oslo.context>=2.19.2 # Apache-2.0
-oslo.db>=8.4.0 # Apache-2.0
+oslo.db>=11.0.0 # Apache-2.0
 oslo.i18n>=3.15.3 # Apache-2.0
-oslo.log>=3.36.0 # Apache-2.0
-oslo.messaging>=5.29.0 # Apache-2.0
-oslo.reports>=1.18.0 # Apache-2.0
-oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
+oslo.log>=4.6.1 # Apache-2.0
+oslo.messaging>=12.5.0 # Apache-2.0
+oslo.reports>=2.2.0 # Apache-2.0
+oslo.serialization>=4.2.0 # Apache-2.0
 oslo.upgradecheck>=1.3.0 # Apache-2.0
-oslo.utils>=3.33.0 # Apache-2.0
-oslo.policy>=3.6.0 # Apache-2.0
+oslo.utils>=4.12.1 # Apache-2.0
+oslo.policy>=3.8.1 # Apache-2.0
 osprofiler>=1.4.0 # Apache-2.0
 SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.3.19 # MIT
 autobahn>=22.3.2 # MIT License
--- a/tools/test-setup.sh
+++ b/tools/test-setup.sh
@ -31,3 +31,22 @@ mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e "
    SET default_storage_engine=MYISAM;
    DROP DATABASE IF EXISTS openstack_citest;
    CREATE DATABASE openstack_citest CHARACTER SET utf8;"
+
+# TO fix the mongodb issue in ubuntu 22.04
+ubuntu_version=`cat /etc/issue | cut -d " "  -f2`
+if [[ $ubuntu_version > '22' ]]; then
+    wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add -
+    echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list
+    sudo apt update
+    curl -LO http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
+    sudo dpkg -i ./libssl1.1_1.1.1-1ubuntu2.1~18.04.20_amd64.deb
+    sudo apt install -y mongodb-org
+    sudo systemctl restart mongod
+else
+    sudo apt-get install -y mongodb
+    sudo systemctl restart mongodb
+fi
+
+sudo apt install -y pip
+pip install setuptools
+sudo python3 setup.py install
--- a/tox.ini
+++ b/tox.ini
@ -17,13 +17,11 @@ setenv = VIRTUAL_ENV={envdir}
         OS_STDOUT_CAPTURE=1
         OS_STDERR_CAPTURE=1
         OS_TEST_TIMEOUT=60
-deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-       -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
+deps =
+  -r{toxinidir}/test-requirements.txt
+  -r{toxinidir}/requirements.txt
 commands =
-    find . -type f -name "*.pyc" -delete
    stestr run --serial --slowest {posargs}
-allowlist_externals = find

 [testenv:integration]
 setenv = {[testenv]setenv}
@ -31,6 +29,10 @@ setenv = {[testenv]setenv}
         OS_TEST_PATH=./zaqar/tests/functional
 commands = stestr run --serial --slowest {posargs}

+[testenv:py{3,38,39,310,311}]
+install_command =
+  python -m pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} {opts} {packages}
+
 [testenv:pep8]
 commands =
  doc8 doc/source
@ -58,8 +60,10 @@ commands =
 commands = {posargs}

 [testenv:docs]
+install_command =
+    python -m pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} {opts} {packages}
 deps =
-    -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+    -r{toxinidir}/requirements.txt
    -r{toxinidir}/doc/requirements.txt
 commands =
    sphinx-build -W -b html doc/source doc/build/html
--- a/zaqar/storage/mongodb/flavors.py
+++ b/zaqar/storage/mongodb/flavors.py
@ -46,12 +46,17 @@ class FlavorsController(base.FlavorsBase):

    def __init__(self, *args, **kwargs):
        super(FlavorsController, self).__init__(*args, **kwargs)
-
+        # To avoid creating unique index twice
+        flavors_index_str = '_'.join(
+            map(lambda x: '%s_%s' % (x[0], x[1]), FLAVORS_INDEX)
+        )
        self._col = self.driver.database.flavors
-        self._col.create_index(FLAVORS_INDEX,
-                               background=True,
-                               name='flavors_name',
-                               unique=True)
+        indexes = self._col.index_information().keys()
+        if flavors_index_str and flavors_index_str not in indexes:
+            self._col.create_index(FLAVORS_INDEX,
+                                   background=True,
+                                   name='flavors_name',
+                                   unique=True)
        self._col.create_index(FLAVORS_STORAGE_POOL_INDEX,
                               background=True,
                               name='flavors_storage_pool_group_name')
--- a/zaqar/storage/mongodb/pools.py
+++ b/zaqar/storage/mongodb/pools.py
@ -55,17 +55,25 @@ class PoolsController(base.PoolsBase):

    def __init__(self, *args, **kwargs):
        super(PoolsController, self).__init__(*args, **kwargs)
-
+        # To avoid creating unique index twice
+        pools_index_str = '_'.join(
+            map(lambda x: '%s_%s' % (x[0], x[1]), POOLS_INDEX)
+        )
+        uri_index_str = '_'.join(
+            map(lambda x: '%s_%s' % (x[0], x[1]), URI_INDEX)
+        )
        self._col = self.driver.database.pools
-        self._col.create_index(POOLS_INDEX,
-                               background=True,
-                               name='pools_name',
-                               unique=True)
-
-        self._col.create_index(URI_INDEX,
-                               background=True,
-                               name='pools_uri',
-                               unique=True)
+        indexes = self._col.index_information().keys()
+        if pools_index_str and pools_index_str not in indexes:
+            self._col.create_index(POOLS_INDEX,
+                                   background=True,
+                                   name='pools_name',
+                                   unique=True)
+        if uri_index_str and uri_index_str not in indexes:
+            self._col.create_index(URI_INDEX,
+                                   background=True,
+                                   name='pools_uri',
+                                   unique=True)

    @utils.raises_conn_error
    def _list(self, marker=None, limit=10, detailed=False):
--- a/zaqar/storage/utils.py
+++ b/zaqar/storage/utils.py
@ -19,7 +19,7 @@ from oslo_config import cfg
 from oslo_log import log
 from oslo_serialization import jsonutils
 from osprofiler import profiler
-from stevedore import driver
+import stevedore
 from urllib import parse as urllib_parse

 from zaqar.common import errors
@ -94,8 +94,9 @@ def load_storage_impl(uri, control_mode=False, default_store=None):
        storage_type = default_store

    try:
-        mgr = driver.DriverManager(driver_type, storage_type,
-                                   invoke_on_load=False)
+        mgr = stevedore.DriverManager(namespace=driver_type,
+                                      name=storage_type,
+                                      invoke_on_load=False)

        return mgr.driver

@ -138,10 +139,10 @@ def load_storage_driver(conf, cache, storage_type=None,
        _invoke_args = (conf, cache, control_driver)

    try:
-        mgr = driver.DriverManager(driver_type,
-                                   storage_type,
-                                   invoke_on_load=True,
-                                   invoke_args=_invoke_args)
+        mgr = stevedore.DriverManager(namespace=driver_type,
+                                      name=storage_type,
+                                      invoke_on_load=True,
+                                      invoke_args=_invoke_args)

        if conf.profiler.enabled:
            if ((mode == "control" and conf.profiler.trace_management_store) or