add option for adding a suffix to output files

When we run update.py automatically in the devstack gate, we
have the issue that we are programatically changing files which
are actually git versioned. This makes lots of people sad pandas.

As this is all being done automatically we could actually update
the files with a suffix, then have devstack use that instead.
This will make developers happy, as they won't have to keep
resetting their devstack trees if they do development straight in
them.

Change-Id: I3d264f15ab7123f258f198b375b9667895c4104c

tests/test_update_suffix.py

@@ -0,0 +1,107 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from __future__ import print_function
+
+import os
+import os.path
+import shutil
+import subprocess
+import sys
+import tempfile
+import testtools
+
+
+def _file_to_list(fname):
+    with open(fname) as f:
+        content = map(lambda x: x.rstrip(), f.readlines())
+        print(content)
+        return content
+
+
+class UpdateTestWithSuffix(testtools.TestCase):
+
+    def setUp(self):
+        super(UpdateTestWithSuffix, self).setUp()
+        self.dir = tempfile.mkdtemp()
+        self.project_dir = os.path.join(self.dir, "project")
+        self.oslo_dir = os.path.join(self.dir, "project_with_oslo")
+
+        self.req_file = os.path.join(self.dir, "global-requirements.txt")
+        self.dev_req_file = os.path.join(self.dir, "dev-requirements.txt")
+        self.proj_file = os.path.join(self.project_dir, "requirements.txt")
+        self.oslo_file = os.path.join(self.oslo_dir, "requirements.txt")
+        self.proj_test_file = os.path.join(self.project_dir,
+                                           "test-requirements.txt")
+        self.setup_file = os.path.join(self.project_dir, "setup.py")
+        self.old_setup_file = os.path.join(self.oslo_dir, "setup.py")
+        os.mkdir(self.project_dir)
+        os.mkdir(self.oslo_dir)
+
+        shutil.copy("tests/files/gr-base.txt", self.req_file)
+        shutil.copy("tests/files/dev-req.txt", self.dev_req_file)
+        shutil.copy("tests/files/project-with-oslo-tar.txt", self.oslo_file)
+        shutil.copy("tests/files/project.txt", self.proj_file)
+        shutil.copy("tests/files/test-project.txt", self.proj_test_file)
+        shutil.copy("tests/files/setup.py", self.setup_file)
+        shutil.copy("tests/files/old-setup.py", self.old_setup_file)
+        shutil.copy("update.py", os.path.join(self.dir, "update.py"))
+
+        # now go call update and see what happens
+        self.addCleanup(os.chdir, os.path.abspath(os.curdir))
+        os.chdir(self.dir)
+        subprocess.call([sys.executable, "update.py",
+                         "-o", "global", "project"])
+        subprocess.call([sys.executable, "update.py",
+                         "-o", "global", "project_with_oslo"])
+
+    def test_requirements(self):
+        # this is the sanity check test
+        reqs = _file_to_list(self.req_file)
+        self.assertIn("jsonschema>=1.0.0,!=1.4.0,<2", reqs)
+
+    def test_project(self):
+        reqs = _file_to_list("%s.%s" % (self.proj_file, 'global'))
+        # ensure various updates take
+        self.assertIn("jsonschema>=1.0.0,!=1.4.0,<2", reqs)
+        self.assertIn("python-keystoneclient>=0.3.0", reqs)
+        self.assertIn("SQLAlchemy>=0.7,<=0.7.99", reqs)
+
+    def test_project_with_oslo(self):
+        reqs = _file_to_list("%s.%s" % (self.oslo_file, 'global'))
+        oslo_tar = ("-f http://tarballs.openstack.org/oslo.config/"
+                    "oslo.config-1.2.0a3.tar.gz#egg=oslo.config-1.2.0a3")
+        self.assertIn(oslo_tar, reqs)
+        self.assertIn("oslo.config>=1.2.0a3", reqs)
+        self.assertNotIn("oslo.config>=1.1.0", reqs)
+
+    def test_test_project(self):
+        reqs = _file_to_list("%s.%s" % (self.proj_test_file, 'global'))
+        self.assertIn("testtools>=0.9.32", reqs)
+        self.assertIn("testrepository>=0.0.17", reqs)
+        # make sure we didn't add something we shouldn't
+        self.assertNotIn("sphinxcontrib-pecanwsme>=0.2", reqs)
+
+    def test_install_setup(self):
+        setup_contents = _file_to_list(self.setup_file)
+        self.assertIn("# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO"
+                      " - DO NOT EDIT", setup_contents)
+
+    def test_no_install_setup(self):
+        setup_contents = _file_to_list(self.old_setup_file)
+        self.assertNotIn(
+            "# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO"
+            " - DO NOT EDIT", setup_contents)

update.py

@@ -28,6 +28,7 @@ updated to match the global requirements. Requirements not in the global
 files will be dropped.
 """
 
+import optparse
 import os
 import os.path
 import sys
@@ -92,11 +93,16 @@ def _parse_reqs(filename):
     return reqs
 
 
-def _sync_requirements_file(source_reqs, dev_reqs, dest_path):
+def _sync_requirements_file(source_reqs, dev_reqs, dest_path, suffix):
     dest_reqs = []
     with open(dest_path, 'r') as dest_reqs_file:
         dest_reqs = dest_reqs_file.readlines()
 
+    # this is specifically for global-requirements gate jobs so we don't
+    # modify the git tree
+    if suffix:
+        dest_path = "%s.%s" % (dest_path, suffix)
+
     print("Syncing %s" % dest_path)
 
     with open(dest_path, 'w') as new_reqs:
@@ -124,7 +130,7 @@ def _sync_requirements_file(source_reqs, dev_reqs, dest_path):
                     new_reqs.write("%s\n" % source_reqs[old_pip])
 
 
-def _copy_requires(dest_dir):
+def _copy_requires(suffix, dest_dir):
     """Copy requirements files."""
 
     source_reqs = _parse_reqs('global-requirements.txt')
@@ -139,7 +145,7 @@ def _copy_requires(dest_dir):
         if os.path.exists(dest_path):
             print("_sync_requirements_file(%s, %s, %s)" %
                   (source_reqs, dev_reqs, dest_path))
-            _sync_requirements_file(source_reqs, dev_reqs, dest_path)
+            _sync_requirements_file(source_reqs, dev_reqs, dest_path, suffix)
 
 
 def _write_setup_py(dest_path):
@@ -152,10 +158,17 @@ def _write_setup_py(dest_path):
             setup_file.write(_setup_py_text)
 
 
-def main(argv):
-    _copy_requires(argv[0])
-    _write_setup_py(argv[0])
+def main(options, args):
+    if len(args) != 1:
+        print("Must specify directory to update")
+        sys.exit(1)
+    _copy_requires(options.suffix, args[0])
+    _write_setup_py(args[0])
 
 
 if __name__ == "__main__":
-    main(sys.argv[1:])
+    parser = optparse.OptionParser()
+    parser.add_option("-o", "--output-suffix", dest="suffix", default="",
+                      help="output suffix for updated files (i.e. .global)")
+    (options, args) = parser.parse_args()
+    main(options, args)