diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c6f317d --- /dev/null +++ b/.gitignore @@ -0,0 +1,61 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg* +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +cover/ +.coverage* +!.coveragerc +.tox +nosetests.xml +.stestr/ +.venv +testenv + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build +doc/source/reference/api + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp +.*sw? + +# Files created by releasenotes build +releasenotes/build + diff --git a/.zuul.yaml b/.zuul.yaml index 0b68faa..97f521e 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,4 +1,13 @@ - project: templates: - - openstack-python3-ussuri-jobs - - check-requirements + #NOTE(rpittau): to run the requirements test we need an alternative to + # daemonize as it's unmaintained since 2018 and it doesn't support + # python >= 3.6 + #- check-requirements + #NOTE(rpittau): cover tests needs some config changes, we'll enable them + # in a future patch + #- openstack-cover-jobs + - openstack-lower-constraints-master-branch-jobs + - openstack-python3-yoga-jobs + - publish-openstack-docs-pti + - release-notes-jobs-python3 diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..d8817fb --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,7 @@ +openstackdocstheme>=2.2.0 # Apache-2.0 +os-api-ref>=1.4.0 # Apache-2.0 +reno>=3.1.0 # Apache-2.0 +sphinx>=2.0.0,!=2.1.0 # BSD +sphinxcontrib-apidoc>=0.2.0 # BSD +sphinxcontrib-seqdiag>=0.8.4 # BSD +sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD \ No newline at end of file diff --git a/doc/source/conf.py b/doc/source/conf.py index 8f3541a..2f1f670 100755 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -37,7 +37,7 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'Molten Iron' +project = u'molteniron' copyright = u'2016, OpenStack Foundation' # If true, '()' will be appended to :func: etc. cross-reference text. @@ -48,7 +48,12 @@ add_function_parentheses = True add_module_names = True # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = 'native' + +# openstackdocstheme options +openstackdocs_repo_name = 'openstack/molteniron' +openstackdocs_use_storyboard = True +openstackdocs_pdf_link = True # -- Options for HTML output -------------------------------------------------- diff --git a/lower-constraints.txt b/lower-constraints.txt new file mode 100644 index 0000000..024eddf --- /dev/null +++ b/lower-constraints.txt @@ -0,0 +1,9 @@ +PyMySQL==0.8.0 +PyYAML==3.10 +SQLAlchemy-Utils==0.30.11 +SQLAlchemy==1.2.19 +coverage==4.0 +daemonize==2.5.0 +oslotest==3.2.0 +pbr==2.0.0 +stestr==2.0.0 \ No newline at end of file diff --git a/molteniron/molteniron b/molteniron/molteniron index 98471ca..692ab7c 100755 --- a/molteniron/molteniron +++ b/molteniron/molteniron @@ -25,12 +25,14 @@ a MoltenIron server. import argparse import json -from molteniron import molteniron import os -from pkg_resources import resource_filename import sys + +from pkg_resources import resource_filename import yaml +from molteniron import molteniron + if __name__ == "__main__": mi = molteniron.MoltenIron() diff --git a/molteniron/molteniron.py b/molteniron/molteniron.py index a2e3b6d..be9cac8 100644 --- a/molteniron/molteniron.py +++ b/molteniron/molteniron.py @@ -45,6 +45,7 @@ def makeRegistrar(): registrar.all = registry return registrar + # Create the decorator command = makeRegistrar() diff --git a/molteniron/moltenirond-helper b/molteniron/moltenirond-helper index efab8c6..4e684b3 100755 --- a/molteniron/moltenirond-helper +++ b/molteniron/moltenirond-helper @@ -23,14 +23,17 @@ This is a helper program for the MoltenIron server. # pylint: disable=redefined-outer-name import argparse -from daemonize import Daemonize -from molteniron import moltenirond import os -from pkg_resources import resource_filename import signal import sys + +from daemonize import Daemonize +from pkg_resources import resource_filename import yaml +from molteniron import moltenirond + + PID = "/var/run/moltenirond.pid" YAML_CONF = None ERROR_LOGFILE = "/tmp/MoltenIron-error-logfile" @@ -156,8 +159,8 @@ if __name__ == "__main__": daemon = Daemonize(app="moltenirond", pid=PID, action=moltenirond_main) - daemon.start() + elif args.command[0].upper().lower() == "stop": try: pid = get_moltenirond_pid() diff --git a/molteniron/moltenirond.py b/molteniron/moltenirond.py index 101e245..6314316 100755 --- a/molteniron/moltenirond.py +++ b/molteniron/moltenirond.py @@ -29,17 +29,16 @@ This is the MoltenIron server. import argparse import calendar +import collections +from contextlib import contextmanager from datetime import datetime import json import os -from pkg_resources import resource_filename import sys import time import traceback -import yaml - -from contextlib import contextmanager +from pkg_resources import resource_filename from sqlalchemy import create_engine from sqlalchemy import Column, Integer, String, ForeignKey from sqlalchemy.exc import InternalError, OperationalError @@ -49,10 +48,8 @@ from sqlalchemy.schema import MetaData, Table from sqlalchemy.sql import insert, update, delete from sqlalchemy.sql import and_ from sqlalchemy.types import TIMESTAMP - import sqlalchemy_utils - -import collections # noqa +import yaml if sys.version_info >= (3, 0): from http.server import HTTPServer, BaseHTTPRequestHandler # noqa @@ -229,8 +226,8 @@ class Nodes(declarative_base()): """Returns a map of the database row contents""" return {key: value for key, value in list(self.__dict__.items()) - if not key.startswith('_') and - not isinstance(key, collections.Callable)} + if not key.startswith('_') + and not isinstance(key, collections.Callable)} def __repr__(self): fmt = """""" self.node_id, self.ip) + TYPE_MYSQL = 1 # Is there a mysql memory path? TYPE_SQLITE = 3 @@ -1039,10 +1037,7 @@ class DataBase(object): for (field, length, _, skip) in ei: if skip: continue - dl += (" " + - field + - ' ' * (length - len(field)) + - " +") + dl += (" " + field + ' ' * (length - len(field)) + " +") index = 0 fl = "|" diff --git a/molteniron/tests/testAddBMNode.py b/molteniron/tests/testAddBMNode.py index 360c620..f764f66 100755 --- a/molteniron/tests/testAddBMNode.py +++ b/molteniron/tests/testAddBMNode.py @@ -22,12 +22,15 @@ Tests the addBMNode MoltenIron command. # pylint: disable-msg=C0103 import argparse -from molteniron import moltenirond import os -from pkg_resources import resource_filename import sys + +from pkg_resources import resource_filename import yaml +from molteniron import moltenirond + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Molteniron CLI tool") parser.add_argument("-c", diff --git a/molteniron/tests/testAllocateBM.py b/molteniron/tests/testAllocateBM.py index 19bfb93..d88cf00 100755 --- a/molteniron/tests/testAllocateBM.py +++ b/molteniron/tests/testAllocateBM.py @@ -23,12 +23,14 @@ Tests the MoltenIron allocateBM command. import argparse import json -from molteniron import moltenirond import os -from pkg_resources import resource_filename import sys + +from pkg_resources import resource_filename import yaml +from molteniron import moltenirond + def result_to_r(res): """Takes a result and returns the request parameters.""" diff --git a/molteniron/tests/testCull.py b/molteniron/tests/testCull.py index 26f9d1f..367b022 100755 --- a/molteniron/tests/testCull.py +++ b/molteniron/tests/testCull.py @@ -23,13 +23,15 @@ Tests the MoltenIron cull command. import argparse import json -from molteniron import moltenirond import os -from pkg_resources import resource_filename import sys import time + +from pkg_resources import resource_filename import yaml +from molteniron import moltenirond + def result_to_r(res): """Takes a result and returns the request parameters.""" @@ -68,6 +70,7 @@ def compare_culled_nodes(lhs, rhs_r, rhs_n): assert lhs_r == rhs_r assert lhs_n == rhs_n + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Molteniron CLI tool") parser.add_argument("-c", diff --git a/molteniron/tests/testDeallocateBM.py b/molteniron/tests/testDeallocateBM.py index 3f435a3..fd483da 100755 --- a/molteniron/tests/testDeallocateBM.py +++ b/molteniron/tests/testDeallocateBM.py @@ -23,12 +23,14 @@ Tests the MoltenIron deallocateBM command. import argparse import json -from molteniron import moltenirond import os -from pkg_resources import resource_filename import sys + +from pkg_resources import resource_filename import yaml +from molteniron import moltenirond + def result_to_r(res): """Takes a result and returns the request parameters.""" diff --git a/molteniron/tests/testDeallocateOwner.py b/molteniron/tests/testDeallocateOwner.py index 449ba44..d0e3058 100755 --- a/molteniron/tests/testDeallocateOwner.py +++ b/molteniron/tests/testDeallocateOwner.py @@ -23,12 +23,14 @@ Tests the MoltenIron deallocateOwner command. import argparse import json -from molteniron import moltenirond import os -from pkg_resources import resource_filename import sys + +from pkg_resources import resource_filename import yaml +from molteniron import moltenirond + def result_to_r(res): """Takes a result and returns the request parameters.""" diff --git a/molteniron/tests/testDoClean.py b/molteniron/tests/testDoClean.py index b0613c1..88e73d0 100755 --- a/molteniron/tests/testDoClean.py +++ b/molteniron/tests/testDoClean.py @@ -22,12 +22,14 @@ Tests the MoltenIron doClean command. # pylint: disable-msg=C0103 import argparse -from molteniron import moltenirond import os -from pkg_resources import resource_filename import sys + +from pkg_resources import resource_filename import yaml +from molteniron import moltenirond + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Molteniron CLI tool") diff --git a/molteniron/tests/testGetField.py b/molteniron/tests/testGetField.py index 397ac1c..dfcad2c 100755 --- a/molteniron/tests/testGetField.py +++ b/molteniron/tests/testGetField.py @@ -22,12 +22,14 @@ Tests the MoltenIron get_field command. # pylint: disable-msg=C0103 import argparse -from molteniron import moltenirond import os -from pkg_resources import resource_filename import sys + +from pkg_resources import resource_filename import yaml +from molteniron import moltenirond + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Molteniron CLI tool") diff --git a/molteniron/tests/testGetIps.py b/molteniron/tests/testGetIps.py index 1b7c344..51335ef 100755 --- a/molteniron/tests/testGetIps.py +++ b/molteniron/tests/testGetIps.py @@ -22,12 +22,14 @@ Tests the MoltenIron get_ips command. # pylint: disable-msg=C0103 import argparse -from molteniron import moltenirond import os -from pkg_resources import resource_filename import sys + +from pkg_resources import resource_filename import yaml +from molteniron import moltenirond + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Molteniron CLI tool") diff --git a/molteniron/tests/testRemoveBMNode.py b/molteniron/tests/testRemoveBMNode.py index 5fb63d5..af703ae 100755 --- a/molteniron/tests/testRemoveBMNode.py +++ b/molteniron/tests/testRemoveBMNode.py @@ -22,12 +22,14 @@ Tests the MoltenIron removeBMNode command. # pylint: disable-msg=C0103 import argparse -from molteniron import moltenirond import os -from pkg_resources import resource_filename import sys + +from pkg_resources import resource_filename import yaml +from molteniron import moltenirond + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Molteniron CLI tool") diff --git a/requirements.txt b/requirements.txt index 5f2815a..1eaa2ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,10 +2,9 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -pbr>=1.6 # Apache-2.0 - -daemonize -PyMySQL>=0.6.2,!=0.7.7 # MIT License -pyyaml -sqlalchemy -sqlalchemy_utils +pbr>=2.0.0 # Apache-2.0 +daemonize>=2.5.0 # MIT License +PyMySQL>=0.8.0 # MIT License +PyYAML>=3.10 # MIT License +SQLAlchemy>=1.2.19 # MIT License +SQLAlchemy-Utils>=0.30.11 # BSD diff --git a/setup.cfg b/setup.cfg index 07b03c4..0b2c2de 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,7 @@ classifier = Programming Language :: Python :: 3 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 Programming Language :: Python :: 3 :: Only Programming Language :: Python :: Implementation :: CPython diff --git a/test-requirements.txt b/test-requirements.txt index 761b0dd..16c5d48 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,16 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=0.11.0,<0.12 # Apache-2.0 - -coverage>=4.0 # Apache-2.0 -python-subunit>=0.0.18 # Apache-2.0/BSD -sphinx>=1.2.1,!=1.3b1,<1.4 # BSD -openstackdocstheme>=1.11.0 # Apache-2.0 -oslotest>=1.10.0 # Apache-2.0 -testrepository>=0.0.18 # Apache-2.0/BSD -testscenarios>=0.4 # Apache-2.0/BSD -testtools>=1.4.0 # MIT - -# releasenotes -reno>=1.8.0 # Apache2 +# unit tests +coverage!=4.4,>=4.0 # Apache-2.0 +oslotest>=3.2.0 # Apache-2.0 +stestr>=2.0.0 # Apache-2.0 diff --git a/tools/test-setup.sh b/tools/test-setup.sh index 07a0785..505a58c 100755 --- a/tools/test-setup.sh +++ b/tools/test-setup.sh @@ -23,8 +23,8 @@ sudo -H mysqladmin -u root password $DB_ROOT_PW sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e " DELETE FROM mysql.user WHERE User=''; FLUSH PRIVILEGES; - GRANT ALL PRIVILEGES ON *.* - TO '$DB_USER'@'%' identified by '$DB_PW' WITH GRANT OPTION;" + CREATE USER '$DB_USER'@'%' IDENTIFIED BY '$DB_PW'; + GRANT ALL PRIVILEGES ON *.* TO '$DB_USER'@'%' WITH GRANT OPTION;" # Now create our database. mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " diff --git a/tox.ini b/tox.ini index 1e911c9..5f49073 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] -minversion = 3.1.0 -envlist = py3,pypy,pep8 -ignore_basepython_conflict = True +minversion = 3.18.0 skipsdist = True +envlist = py3,pep8 +ignore_basepython_conflict=true [testenv] basepython = python3 @@ -11,12 +11,11 @@ setenv = VIRTUAL_ENV={envdir} PYTHONWARNINGS=default::DeprecationWarning deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/test-requirements.txt -# Don't worry about installing bash commands in the virtual environment. -whitelist_externals = mkdir + -r{toxinidir}/requirements.txt +allowlist_externals = mkdir diff -#commands = python setup.py test --slowest --testr-args='{posargs}' commands = mkdir -p testenv/var/run/ mkdir -p testenv/tmp/ python setup.py \ @@ -87,19 +86,54 @@ commands = mkdir -p testenv/var/run/ stop [testenv:pep8] +deps= + hacking>=4.1.0,<5.0.0 # Apache-2.0 + flake8-import-order>=0.17.1 # LGPLv3 + pycodestyle>=2.0.0,<3.0.0 # MIT commands = flake8 {posargs} [testenv:venv] +setenv = PYTHONHASHSEED=0 +deps = + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/doc/requirements.txt commands = {posargs} [testenv:cover] -commands = python setup.py test --coverage --testr-args='{posargs}' +setenv = + {[testenv]setenv} + PYTHON=coverage run --parallel-mode +# After running this target, visit molteniron/cover/index.html +# in your browser, to see a nicer presentation report with annotated +# HTML listings detailing missed lines. +commands = coverage erase + stestr run {posargs} + coverage combine + coverage report + coverage html + coverage xml -o cover/coverage.xml [testenv:docs] +deps = + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/requirements.txt + -r{toxinidir}/doc/requirements.txt +commands = sphinx-build -b html -W doc/source doc/build/html + +[testenv:pdf-docs] +allowlist_externals = make +deps = {[testenv:docs]deps} commands = - sphinx-build -W -d doc/build/doctrees -b html doc/source doc/build/html + sphinx-build -W -b latex doc/source doc/build/pdf + make -C doc/build/pdf [testenv:releasenotes] +usedevelop = False +deps = + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html @@ -107,9 +141,29 @@ commands = commands = oslo_debug_helper {posargs} [flake8] -# E123, E125 skipped as they are invalid PEP-8. - show-source = True -ignore = E123,E125 +# E123, E125 skipped as they are invalid PEP-8. +# E741 ambiguous variable name. +# W503 Line break occurred before a binary operator. Conflicts with W504. +ignore = E123,E125,E741,W503 +# [H106] Don't put vim configuration in source files. +# [H203] Use assertIs(Not)None to check for None. +# [H204] Use assert(Not)Equal to check for equality. +# [H205] Use assert(Greater|Less)(Equal) for comparison. +# [H210] Require 'autospec', 'spec', or 'spec_set' in mock.patch/mock.patch.object calls +# [H904] Delay string interpolations at logging calls. +enable-extensions=H106,H203,H204,H205,H210,H904 builtins = _ -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,testenv +import-order-style = pep8 +application-import-names = molteniron +filename = + *.py, + *molteniron, + *moltenirond-helper + +[testenv:lower-constraints] +deps = + -c{toxinidir}/lower-constraints.txt + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements.txt diff --git a/utils/createDB.py b/utils/createDB.py index 394a6cf..490f88b 100755 --- a/utils/createDB.py +++ b/utils/createDB.py @@ -23,6 +23,7 @@ Create the MoltenIron user in mysql and grant it access. import os import sys + import yaml @@ -42,19 +43,19 @@ def main(): conf = yaml.load(fobj) # Create the SQL User - SQL("CREATE USER '" + - conf["sqlUser"] + - "'@'localhost' IDENTIFIED BY '" + - conf["sqlPass"] + - "';") + SQL("CREATE USER '" + + conf["sqlUser"] + + "'@'localhost' IDENTIFIED BY '" + + conf["sqlPass"] + "';") # And grant that SQL user access to the MoltenIron database - SQL("GRANT ALL ON MoltenIron.* TO '" + - conf["sqlUser"] + - "'@'localhost';") + SQL("GRANT ALL ON MoltenIron.* TO '" + + conf["sqlUser"] + + "'@'localhost';") return 0 + if __name__ == "__main__": rc = main() diff --git a/utils/test_hook_mi_ipmiblob.py b/utils/test_hook_mi_ipmiblob.py index cafd130..8f658f7 100755 --- a/utils/test_hook_mi_ipmiblob.py +++ b/utils/test_hook_mi_ipmiblob.py @@ -25,12 +25,14 @@ a MoltenIron server. import argparse import json -from molteniron import molteniron import os -from pkg_resources import resource_filename import sys + +from pkg_resources import resource_filename import yaml +from molteniron import molteniron + if __name__ == "__main__": mi = molteniron.MoltenIron() @@ -107,8 +109,8 @@ if __name__ == "__main__": try: rc = response_map["status"] except KeyError: - msg = ("Error: Server returned: %s and we expected a status " + - "somewhere") % (response_map, ) + msg = ("Error: Server returned: %s and we expected a status " + + "somewhere") % (response_map, ) print(msg, file=sys.stderr) exit(444) @@ -150,10 +152,10 @@ if __name__ == "__main__": with open(args.hardware_info, "w") as hi_obj: # Write one line - hi_obj.write(("%(ipmi_ip)s" % node) + - (" %(port_hwaddr)s" + - " %(ipmi_user)s" + - " %(ipmi_password)s\n") % blob) + hi_obj.write(("%(ipmi_ip)s" % node) + + (" %(port_hwaddr)s" + + " %(ipmi_user)s" + + " %(ipmi_password)s\n") % blob) pool = node["allocation_pool"].split(",") @@ -162,8 +164,8 @@ if __name__ == "__main__": with open(args.localrc, "a") as l_obj: # Write multiple lines - l_obj.write(("IRONIC_HW_ARCH=%(cpu_arch)s\n" + - "IRONIC_HW_NODE_CPU=%(cpus)s\n" + - "IRONIC_HW_NODE_RAM=%(ram_mb)s\n" + - "IRONIC_HW_NODE_DISK=%(disk_gb)s\n") % blob + - "ALLOCATION_POOL=\"%s\"\n" % (allocation_pool, )) + l_obj.write(("IRONIC_HW_ARCH=%(cpu_arch)s\n" + + "IRONIC_HW_NODE_CPU=%(cpus)s\n" + + "IRONIC_HW_NODE_RAM=%(ram_mb)s\n" + + "IRONIC_HW_NODE_DISK=%(disk_gb)s\n") % blob + + "ALLOCATION_POOL=\"%s\"\n" % (allocation_pool, ))