Update molteniron tests and requirements

Make things as pretty as possible while moving to current openstack
Y cycle tests.

Change-Id: I22777cf34888171daf69c44705d8ecc7339f8278

.gitignore

@@ -0,0 +1,61 @@
+*.py[cod]
+
+# C extensions
+*.so
+
+# Packages
+*.egg*
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+cover/
+.coverage*
+!.coveragerc
+.tox
+nosetests.xml
+.stestr/
+.venv
+testenv
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Complexity
+output/*.html
+output/*/index.html
+
+# Sphinx
+doc/build
+doc/source/reference/api
+
+# pbr generates these
+AUTHORS
+ChangeLog
+
+# Editors
+*~
+.*.swp
+.*sw?
+
+# Files created by releasenotes build
+releasenotes/build
+

.zuul.yaml

@@ -1,4 +1,13 @@
 - project:
     templates:
-      - openstack-python3-ussuri-jobs
+      #NOTE(rpittau): to run the requirements test we need an alternative to
-      - check-requirements
+      # daemonize as it's unmaintained since 2018 and it doesn't support
+      # python >= 3.6
+      #- check-requirements
+      #NOTE(rpittau): cover tests needs some config changes, we'll enable them
+      # in a future patch
+      #- openstack-cover-jobs
+      - openstack-lower-constraints-master-branch-jobs
+      - openstack-python3-yoga-jobs
+      - publish-openstack-docs-pti
+      - release-notes-jobs-python3

doc/requirements.txt

@@ -0,0 +1,7 @@
+openstackdocstheme>=2.2.0 # Apache-2.0
+os-api-ref>=1.4.0 # Apache-2.0
+reno>=3.1.0 # Apache-2.0
+sphinx>=2.0.0,!=2.1.0 # BSD
+sphinxcontrib-apidoc>=0.2.0  # BSD
+sphinxcontrib-seqdiag>=0.8.4 # BSD
+sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD

doc/source/conf.py

@@ -37,7 +37,7 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-project = u'Molten Iron'
+project = u'molteniron'
 copyright = u'2016, OpenStack Foundation'
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
@@ -48,7 +48,12 @@ add_function_parentheses = True
 add_module_names = True
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'native'
+
+# openstackdocstheme options
+openstackdocs_repo_name = 'openstack/molteniron'
+openstackdocs_use_storyboard = True
+openstackdocs_pdf_link = True
 
 # -- Options for HTML output --------------------------------------------------
 

lower-constraints.txt

@@ -0,0 +1,9 @@
+PyMySQL==0.8.0
+PyYAML==3.10
+SQLAlchemy-Utils==0.30.11
+SQLAlchemy==1.2.19
+coverage==4.0
+daemonize==2.5.0
+oslotest==3.2.0
+pbr==2.0.0
+stestr==2.0.0

molteniron/molteniron

@@ -25,12 +25,14 @@ a MoltenIron server.
 
 import argparse
 import json
-from molteniron import molteniron
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import molteniron
+
 
 if __name__ == "__main__":
     mi = molteniron.MoltenIron()

molteniron/molteniron.py

@@ -45,6 +45,7 @@ def makeRegistrar():
     registrar.all = registry
     return registrar
 
+
 # Create the decorator
 command = makeRegistrar()
 

molteniron/moltenirond-helper

@@ -23,14 +23,17 @@ This is a helper program for the MoltenIron server.
 # pylint: disable=redefined-outer-name
 
 import argparse
-from daemonize import Daemonize
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import signal
 import sys
+
+from daemonize import Daemonize
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
+
 PID = "/var/run/moltenirond.pid"
 YAML_CONF = None
 ERROR_LOGFILE = "/tmp/MoltenIron-error-logfile"
@@ -156,8 +159,8 @@ if __name__ == "__main__":
         daemon = Daemonize(app="moltenirond",
                            pid=PID,
                            action=moltenirond_main)
-
         daemon.start()
+
     elif args.command[0].upper().lower() == "stop":
         try:
             pid = get_moltenirond_pid()

molteniron/moltenirond.py

@@ -29,17 +29,16 @@ This is the MoltenIron server.
 
 import argparse
 import calendar
+import collections
+from contextlib import contextmanager
 from datetime import datetime
 import json
 import os
-from pkg_resources import resource_filename
 import sys
 import time
 import traceback
-import yaml
-
-from contextlib import contextmanager
 
+from pkg_resources import resource_filename
 from sqlalchemy import create_engine
 from sqlalchemy import Column, Integer, String, ForeignKey
 from sqlalchemy.exc import InternalError, OperationalError
@@ -49,10 +48,8 @@ from sqlalchemy.schema import MetaData, Table
 from sqlalchemy.sql import insert, update, delete
 from sqlalchemy.sql import and_
 from sqlalchemy.types import TIMESTAMP
-
 import sqlalchemy_utils
-
+import yaml
-import collections  # noqa
 
 if sys.version_info >= (3, 0):
     from http.server import HTTPServer, BaseHTTPRequestHandler  # noqa
@@ -229,8 +226,8 @@ class Nodes(declarative_base()):
         """Returns a map of the database row contents"""
         return {key: value for key, value
                 in list(self.__dict__.items())
-                if not key.startswith('_') and
+                if not key.startswith('_')
-                not isinstance(key, collections.Callable)}
+                and not isinstance(key, collections.Callable)}
 
     def __repr__(self):
         fmt = """<Node(name='%s',
@@ -291,6 +288,7 @@ ip='%s' />"""
                       self.node_id,
                       self.ip)
 
+
 TYPE_MYSQL = 1
 # Is there a mysql memory path?
 TYPE_SQLITE = 3
@@ -1039,10 +1037,7 @@ class DataBase(object):
         for (field, length, _, skip) in ei:
             if skip:
                 continue
-            dl += (" " +
+            dl += (" " + field + ' ' * (length - len(field)) + " +")
-                   field +
-                   ' ' * (length - len(field)) +
-                   " +")
 
         index = 0
         fl = "|"

molteniron/tests/testAddBMNode.py

@@ -22,12 +22,15 @@ Tests the addBMNode MoltenIron command.
 # pylint: disable-msg=C0103
 
 import argparse
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Molteniron CLI tool")
     parser.add_argument("-c",

molteniron/tests/testAllocateBM.py

@@ -23,12 +23,14 @@ Tests the MoltenIron allocateBM command.
 
 import argparse
 import json
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 def result_to_r(res):
     """Takes a result and returns the request parameters."""

molteniron/tests/testCull.py

@@ -23,13 +23,15 @@ Tests the MoltenIron cull command.
 
 import argparse
 import json
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
 import time
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 def result_to_r(res):
     """Takes a result and returns the request parameters."""
@@ -68,6 +70,7 @@ def compare_culled_nodes(lhs, rhs_r, rhs_n):
     assert lhs_r == rhs_r
     assert lhs_n == rhs_n
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Molteniron CLI tool")
     parser.add_argument("-c",

molteniron/tests/testDeallocateBM.py

@@ -23,12 +23,14 @@ Tests the MoltenIron deallocateBM command.
 
 import argparse
 import json
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 def result_to_r(res):
     """Takes a result and returns the request parameters."""

molteniron/tests/testDeallocateOwner.py

@@ -23,12 +23,14 @@ Tests the MoltenIron deallocateOwner command.
 
 import argparse
 import json
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 def result_to_r(res):
     """Takes a result and returns the request parameters."""

molteniron/tests/testDoClean.py

@@ -22,12 +22,14 @@ Tests the MoltenIron doClean command.
 # pylint: disable-msg=C0103
 
 import argparse
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Molteniron CLI tool")

molteniron/tests/testGetField.py

@@ -22,12 +22,14 @@ Tests the MoltenIron get_field command.
 # pylint: disable-msg=C0103
 
 import argparse
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Molteniron CLI tool")

molteniron/tests/testGetIps.py

@@ -22,12 +22,14 @@ Tests the MoltenIron get_ips command.
 # pylint: disable-msg=C0103
 
 import argparse
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Molteniron CLI tool")

molteniron/tests/testRemoveBMNode.py

@@ -22,12 +22,14 @@ Tests the MoltenIron removeBMNode command.
 # pylint: disable-msg=C0103
 
 import argparse
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Molteniron CLI tool")

requirements.txt

@@ -2,10 +2,9 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-pbr>=1.6 # Apache-2.0
+pbr>=2.0.0 # Apache-2.0
-
+daemonize>=2.5.0 # MIT License
-daemonize
+PyMySQL>=0.8.0 # MIT License
-PyMySQL>=0.6.2,!=0.7.7 # MIT License
+PyYAML>=3.10 # MIT License
-pyyaml
+SQLAlchemy>=1.2.19 # MIT License
-sqlalchemy
+SQLAlchemy-Utils>=0.30.11 # BSD
-sqlalchemy_utils

setup.cfg

@@ -17,6 +17,7 @@ classifier =
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: Implementation :: CPython
 

test-requirements.txt

@@ -2,16 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-hacking>=0.11.0,<0.12 # Apache-2.0
+# unit tests
-
+coverage!=4.4,>=4.0 # Apache-2.0
-coverage>=4.0 # Apache-2.0
+oslotest>=3.2.0 # Apache-2.0
-python-subunit>=0.0.18 # Apache-2.0/BSD
+stestr>=2.0.0 # Apache-2.0
-sphinx>=1.2.1,!=1.3b1,<1.4 # BSD
-openstackdocstheme>=1.11.0 # Apache-2.0
-oslotest>=1.10.0 # Apache-2.0
-testrepository>=0.0.18 # Apache-2.0/BSD
-testscenarios>=0.4 # Apache-2.0/BSD
-testtools>=1.4.0 # MIT
-
-# releasenotes
-reno>=1.8.0 # Apache2

tools/test-setup.sh

@@ -23,8 +23,8 @@ sudo -H mysqladmin -u root password $DB_ROOT_PW
 sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e "
     DELETE FROM mysql.user WHERE User='';
     FLUSH PRIVILEGES;
-    GRANT ALL PRIVILEGES ON *.*
+    CREATE USER '$DB_USER'@'%' IDENTIFIED BY '$DB_PW';
-        TO '$DB_USER'@'%' identified by '$DB_PW' WITH GRANT OPTION;"
+    GRANT ALL PRIVILEGES ON *.* TO '$DB_USER'@'%' WITH GRANT OPTION;"
 
 # Now create our database.
 mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e "

tox.ini

@@ -1,8 +1,8 @@
 [tox]
-minversion = 3.1.0
+minversion = 3.18.0
-envlist = py3,pypy,pep8
-ignore_basepython_conflict = True
 skipsdist = True
+envlist = py3,pep8
+ignore_basepython_conflict=true
 
 [testenv]
 basepython = python3
@@ -11,12 +11,11 @@ setenv =
    VIRTUAL_ENV={envdir}
    PYTHONWARNINGS=default::DeprecationWarning
 deps =
-  -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
   -r{toxinidir}/test-requirements.txt
-# Don't worry about installing bash commands in the virtual environment.
+  -r{toxinidir}/requirements.txt
-whitelist_externals = mkdir
+allowlist_externals = mkdir
                       diff
-#commands = python setup.py test --slowest --testr-args='{posargs}'
 commands = mkdir -p testenv/var/run/
            mkdir -p testenv/tmp/
            python setup.py \
@@ -87,19 +86,54 @@ commands = mkdir -p testenv/var/run/
                stop
 
 [testenv:pep8]
+deps=
+    hacking>=4.1.0,<5.0.0 # Apache-2.0
+    flake8-import-order>=0.17.1 # LGPLv3
+    pycodestyle>=2.0.0,<3.0.0 # MIT
 commands = flake8 {posargs}
 
 [testenv:venv]
+setenv = PYTHONHASHSEED=0
+deps =
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -r{toxinidir}/requirements.txt
+  -r{toxinidir}/test-requirements.txt
+  -r{toxinidir}/doc/requirements.txt
 commands = {posargs}
 
 [testenv:cover]
-commands = python setup.py test --coverage --testr-args='{posargs}'
+setenv =
+   {[testenv]setenv}
+   PYTHON=coverage run --parallel-mode
+# After running this target, visit molteniron/cover/index.html
+# in your browser, to see a nicer presentation report with annotated
+# HTML listings detailing missed lines.
+commands = coverage erase
+           stestr run {posargs}
+           coverage combine
+           coverage report
+           coverage html
+           coverage xml -o cover/coverage.xml
 
 [testenv:docs]
+deps =
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -r{toxinidir}/requirements.txt
+  -r{toxinidir}/doc/requirements.txt
+commands = sphinx-build -b html -W doc/source doc/build/html
+
+[testenv:pdf-docs]
+allowlist_externals = make
+deps = {[testenv:docs]deps}
 commands =
-  sphinx-build -W -d doc/build/doctrees -b html doc/source doc/build/html
+  sphinx-build -W -b latex doc/source doc/build/pdf
+  make -C doc/build/pdf
 
 [testenv:releasenotes]
+usedevelop = False
+deps =
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -r{toxinidir}/doc/requirements.txt
 commands =
   sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
 
@@ -107,9 +141,29 @@ commands =
 commands = oslo_debug_helper {posargs}
 
 [flake8]
-# E123, E125 skipped as they are invalid PEP-8.
-
 show-source = True
-ignore = E123,E125
+# E123, E125 skipped as they are invalid PEP-8.
+# E741 ambiguous variable name.
+# W503 Line break occurred before a binary operator. Conflicts with W504.
+ignore = E123,E125,E741,W503
+# [H106] Don't put vim configuration in source files.
+# [H203] Use assertIs(Not)None to check for None.
+# [H204] Use assert(Not)Equal to check for equality.
+# [H205] Use assert(Greater|Less)(Equal) for comparison.
+# [H210] Require 'autospec', 'spec', or 'spec_set' in mock.patch/mock.patch.object calls
+# [H904] Delay string interpolations at logging calls.
+enable-extensions=H106,H203,H204,H205,H210,H904
 builtins = _
-exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,testenv
+import-order-style = pep8
+application-import-names = molteniron
+filename =
+  *.py,
+  *molteniron,
+  *moltenirond-helper
+
+[testenv:lower-constraints]
+deps =
+  -c{toxinidir}/lower-constraints.txt
+  -r{toxinidir}/test-requirements.txt
+  -r{toxinidir}/requirements.txt

utils/createDB.py

@@ -23,6 +23,7 @@ Create the MoltenIron user in mysql and grant it access.
 
 import os
 import sys
+
 import yaml
 
 
@@ -42,19 +43,19 @@ def main():
     conf = yaml.load(fobj)
 
     # Create the SQL User
-    SQL("CREATE USER '" +
+    SQL("CREATE USER '"
-        conf["sqlUser"] +
+        + conf["sqlUser"]
-        "'@'localhost' IDENTIFIED BY '" +
+        + "'@'localhost' IDENTIFIED BY '"
-        conf["sqlPass"] +
+        + conf["sqlPass"] + "';")
-        "';")
 
     # And grant that SQL user access to the MoltenIron database
-    SQL("GRANT ALL ON MoltenIron.* TO '" +
+    SQL("GRANT ALL ON MoltenIron.* TO '"
-        conf["sqlUser"] +
+        + conf["sqlUser"]
-        "'@'localhost';")
+        + "'@'localhost';")
 
     return 0
 
+
 if __name__ == "__main__":
     rc = main()
 

utils/test_hook_mi_ipmiblob.py

@@ -25,12 +25,14 @@ a MoltenIron server.
 
 import argparse
 import json
-from molteniron import molteniron
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import molteniron
+
 
 if __name__ == "__main__":
     mi = molteniron.MoltenIron()
@@ -107,8 +109,8 @@ if __name__ == "__main__":
         try:
             rc = response_map["status"]
         except KeyError:
-            msg = ("Error: Server returned: %s and we expected a status " +
+            msg = ("Error: Server returned: %s and we expected a status "
-                   "somewhere") % (response_map, )
+                   + "somewhere") % (response_map, )
             print(msg, file=sys.stderr)
             exit(444)
 
@@ -150,10 +152,10 @@ if __name__ == "__main__":
 
         with open(args.hardware_info, "w") as hi_obj:
             # Write one line
-            hi_obj.write(("%(ipmi_ip)s" % node) +
+            hi_obj.write(("%(ipmi_ip)s" % node)
-                         (" %(port_hwaddr)s" +
+                         + (" %(port_hwaddr)s"
-                          " %(ipmi_user)s" +
+                            + " %(ipmi_user)s"
-                          " %(ipmi_password)s\n") % blob)
+                            + " %(ipmi_password)s\n") % blob)
 
         pool = node["allocation_pool"].split(",")
 
@@ -162,8 +164,8 @@ if __name__ == "__main__":
 
         with open(args.localrc, "a") as l_obj:
             # Write multiple lines
-            l_obj.write(("IRONIC_HW_ARCH=%(cpu_arch)s\n" +
+            l_obj.write(("IRONIC_HW_ARCH=%(cpu_arch)s\n"
-                         "IRONIC_HW_NODE_CPU=%(cpus)s\n" +
+                         + "IRONIC_HW_NODE_CPU=%(cpus)s\n"
-                         "IRONIC_HW_NODE_RAM=%(ram_mb)s\n" +
+                         + "IRONIC_HW_NODE_RAM=%(ram_mb)s\n"
-                         "IRONIC_HW_NODE_DISK=%(disk_gb)s\n") % blob +
+                         + "IRONIC_HW_NODE_DISK=%(disk_gb)s\n") % blob
-                        "ALLOCATION_POOL=\"%s\"\n" % (allocation_pool, ))
+                        + "ALLOCATION_POOL=\"%s\"\n" % (allocation_pool, ))