Sync test-requirements with global requirements

In this patch, a tool is provided for team to do the sync to avoid
any gaps with global requirements.

Change-Id: Ie960eae7a2756aa0f267c8ad37ea60f5effe6742
Partial-bug: 1668848

test-requirements.txt

@@ -7,7 +7,7 @@
 coverage>=4.0 # Apache-2.0
 doc8 # Apache-2.0
 fixtures>=3.0.0 # Apache-2.0/BSD
-hacking<0.11,>=0.10.2
+hacking>=0.12.0,!=0.13.0,<0.14 # Apache-2.0
 mock>=2.0 # BSD
 python-subunit>=0.0.18 # Apache-2.0/BSD
 python-tackerclient>=0.8.0 # Apache-2.0

tools/README.rst

@@ -0,0 +1,19 @@
+========================
+Tools for tacker
+========================
+
+sync_test_requirements.py
+========================
+
+Sync the test-requirements.txt with openstack global requirements.
+Assume requirements prject is cloned at /opt/stack/requirements, to run::
+
+    $ git clone https://git.openstack.org/openstack/requirements.git /opt/stack/requirements
+    $ ./sync_test_requirements.py -g /opt/stack/requirements/global-requirements.txt -t ../test-requirements.txt -o ../test-requirements.txt
+
+If this tool shows us the "../test-requirements.txt" is changed,
+please commit it.
+
+This tool is also used in tox.ini to check if the test-requirements.txt is
+synchroized with global-requirements.txt. please refer to tox.ini for the
+usage.

tools/sync_test_requirements.py

@@ -0,0 +1,90 @@
+#!/usr/bin/python
+
+import getopt
+import re
+import sys
+import tempfile
+
+
+def printusage():
+    print('sync_test_requirements.py -g <globalrequirements>'
+          ' -t <testrequirements>'
+          ' [-o <outputfile>]')
+
+
+def main(argv):
+    globareqfile = ''
+    testreqfile = ''
+    outputfile = ''
+    check = False
+    try:
+       opts, args = getopt.getopt(
+           argv,"hg:t:o:",
+           ["globalrequirements=", "testrequirements=", "outputfile="])
+    except getopt.GetoptError:
+       printusage()
+       sys.exit(2)
+
+    for opt, arg in opts:
+       if opt == '-h':
+          printusage()
+          sys.exit()
+       elif opt in ("-g", "--globalrequirements"):
+          globareqfile = arg
+       elif opt in ("-o", "--outputfile"):
+          outputfile = arg
+       elif opt in ("-t", "--testrequirements"):
+          testreqfile = arg
+    if not outputfile:
+        with tempfile.NamedTemporaryFile(delete=False) as tempf:
+            outputfile = tempf.name
+        check = True
+    if not (globareqfile and testreqfile and (outputfile or check)):
+        printusage()
+        sys.exit(2)
+
+    lines = []
+    gmodules = {}
+    changed = 0
+    with open(testreqfile) as testfile:
+        lines = testfile.readlines()
+    with open(globareqfile) as globalfile:
+        globallines = globalfile.readlines()
+        for gline in globallines:
+            gline = gline.rstrip('\n')
+            gmodulename = re.split('[<>=]', gline)[0]
+            moduleparts = gline.split("#")
+            modulepart = moduleparts[0].rstrip(" ")
+            modulelicense = ("" if len(moduleparts) <= 1 else
+                moduleparts[1].strip(" "))
+            gmodules[gmodulename] =(modulepart, modulelicense)
+    with open(outputfile, 'w') as ofile:
+        for line in lines:
+           if line.startswith('#'):
+               ofile.write(line)
+               continue
+           line = line.rstrip('\n')
+           modulename = re.split('[<>=]', line)[0]
+           moduleparts = line.split("#")
+           modulepart = moduleparts[0].rstrip(" ")
+           modulelicense = ("" if len(moduleparts) <= 1 else
+               moduleparts[1].strip(" "))
+           if (gmodules.get(modulename) and
+               (modulepart, modulelicense) != gmodules.get(modulename)):
+               changed = 1
+               if gmodules.get(modulename)[1]:
+                   ofile.write(" # ".join(gmodules.get(modulename)) + "\n")
+               else:
+                   ofile.write(gmodules.get(modulename)[0] + "\n")
+           else:
+               ofile.write(line + "\n")
+    if changed:
+        if check:
+            print("%s is not synchronized with global requirements." %
+                testreqfile)
+        else:
+            print(testreqfile + " is changed.\n")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

tox.ini

@@ -26,6 +26,7 @@ setenv = {[testenv]setenv}
 deps =
   {[testenv:functional]deps}
 commands =
+  {toxinidir}/tools/sync_test_requirements.py -g {toxinidir}/../requirements/global-requirements.txt -t {toxinidir}/test-requirements.txt )
   {toxinidir}/tools/ostestr_compat_shim.sh --concurrency 2 {posargs}
 
 [tox:jenkins]