Finish the Python 3 port

with this change, all unit and functional tests pass on Python 3.4.

* WSGI middleware: on Python 3, decode the error message from UTF-8.
* Replace json.dumps() with jsonutils.dump_as_bytes() of
  oslo_serialization to get bytes on Python 3.
* monitor_service.py, test_monitor.py: Fix usage of tooz, pass byte
  strings to the coordinator API.
* Fix nova and neutron fixtures: replace dict.values() with
  list(dict.values()) to get list on Python 3.
* Fix urllib2_fixture: On Python 3, replace urllib2 with
  urllib.request.
* test_create_cluster.py: don't sort exception types, but use a set
  instead to compare them. On Python 3, comparison between exception
  types now raises a TypeError.
* test_wsme_objects.py: catch also TypeError on wsme validation. On
  Python 3, comparison between the new attribute value and the
  minimum raises a TypeError which is not wrapped into a
  InvalidInput exception of wsme. See also the change
  I74103330ccb5cdc26aa3508fcefcc34310e00c27 which proposes to wrap
  TypeError.
* install_rabbit.sh.tmpl: replace dict.iteritems() with dict.items().
* Replace generator.next() with next(generator).
* Replace reduce() with six.moves.reduce()
* Replace "str is 'URLError'" test with "str == 'URLError'", using is
  to compare strings is not reliable.

Change-Id: I45e23b71218f5ff936b0fd1bd628f29210f984bb

cue/api/middleware/parsable_error.py

@@ -22,10 +22,11 @@ response with one formatted so the client can parse it.
 Based on pecan.middleware.errordocument
 """
 
-import json
 from xml import etree as et
 
 from oslo_log import log
+from oslo_serialization import jsonutils
+import six
 import webob
 
 from cue.common.i18n import _  # noqa
@@ -83,7 +84,11 @@ class ParsableErrorMiddleware(object):
                             + '</error_message>']
                 state['headers'].append(('Content-Type', 'application/xml'))
             else:
-                body = [json.dumps({'error_message': '\n'.join(app_iter)})]
+                err_msg = b'\n'.join(app_iter)
+                if six.PY3:   # pragma: no cover
+                    err_msg = err_msg.decode('utf-8')
+                body = jsonutils.dump_as_bytes({'error_message': err_msg})
+                body = [body]
                 state['headers'].append(('Content-Type', 'application/json'))
             state['headers'].append(('Content-Length', str(len(body[0]))))
         else:

cue/monitor/monitor_service.py

@@ -36,10 +36,10 @@ class MonitorService(service.Service):
                      ))
 
         self.coordinator = coordination.get_coordinator(
-            coord_url, 'cue-monitor')
+            coord_url, b'cue-monitor')
         self.coordinator.start()
         # Create a lock
-        self.lock = self.coordinator.get_lock("status_check")
+        self.lock = self.coordinator.get_lock(b"status_check")
 
     def start(self):
         loop_interval_seconds = int(cfg.CONF.cue_monitor.loop_interval_seconds)

cue/taskflow/retry/exception_times.py

@@ -40,7 +40,7 @@ class ExceptionTimes(retry.Times):
 
     def on_failure(self, history, *args, **kwargs):
 
-        (owner, outcome) = history.outcomes_iter(len(history) - 1).next()
+        (owner, outcome) = next(history.outcomes_iter(len(history) - 1))
 
         if type(outcome.exception) in self._revert_exception_list:
             return self._revert_action

cue/templates/install_rabbit.sh.tmpl

@@ -1,7 +1,7 @@
 #!/bin/bash
 cat > /etc/hosts << EOF
 127.0.0.1 localhost
-{%- for node_name, node_ip in rabbit_nodes.iteritems() %}
+{%- for node_name, node_ip in rabbit_nodes.items() %}
 {{node_ip}} {{node_name}}
 {%- endfor %}
 EOF

cue/tests/functional/fixtures/neutron.py

@@ -134,7 +134,7 @@ class NeutronClient(base.BaseFixture):
         :param retrieve_all: Set to true to retrieve all available ports
         """
         if retrieve_all:
-            return {'ports': self._port_list.values()}
+            return {'ports': list(self._port_list.values())}
 
     def list_networks(self, name=None, id=None):
         if name is not None:

cue/tests/functional/fixtures/nova.py

@@ -309,7 +309,7 @@ class NovaClient(base.BaseFixture):
         :param retrieve_all: Set to true to retrieve all available vms
         """
         if retrieve_all:
-            return self._vm_list.values()
+            return list(self._vm_list.values())
 
     def find_images(self, name, **kwargs):
         """Mock'd version of novaclient...image_find().

cue/tests/functional/fixtures/urllib2_fixture.py

@@ -59,11 +59,16 @@ class Urllib2Fixture(base.BaseFixture):
         """Set up test fixture and apply all method overrides."""
         super(Urllib2Fixture, self).setUp()
 
-        urllib2_client = self.mock('urllib2.OpenerDirector')
+        if six.PY3:
+            modname = 'urllib.request'
+        else:
+            modname = 'urllib2'
+#        modname = 'six.moves.urllib.request'
+        urllib2_client = self.mock('%s.OpenerDirector' % modname)
         urllib2_client.open = self.open
 
     def open(self, url):
         result = Urllib2ResultDetails.get_urllib2_result()
-        if result.getvalue() is 'URLError':
+        if result.getvalue() == 'URLError':
             raise urllib.error.URLError('urlerror')
         return result

cue/tests/functional/taskflow/flow/test_create_cluster.py

@@ -203,10 +203,10 @@ class CreateClusterTests(base.FunctionalTestCase):
         except taskflow_exc.WrappedFailure as err:
             self.assertEqual(3, len(err._causes))
             exc_list = [type(c.exception) for c in err._causes]
-            self.assertEqual(sorted([cue_exceptions.VmErrorException,
-                                    cue_exceptions.VmBuildingException,
-                                    cue_exceptions.VmBuildingException]),
-                             sorted(exc_list))
+            self.assertEqual({cue_exceptions.VmErrorException,
+                              cue_exceptions.VmBuildingException,
+                              cue_exceptions.VmBuildingException},
+                             set(exc_list))
         except Exception as e:
             self.assertEqual(taskflow_exc.WrappedFailure, type(e))
         else:
@@ -261,10 +261,10 @@ class CreateClusterTests(base.FunctionalTestCase):
             engines.run(flow, store=flow_store)
         except taskflow_exc.WrappedFailure as err:
             self.assertEqual(3, len(err._causes))
-            exc_list = [type(c.exception) for c in err._causes]
-            self.assertEqual([cue_exceptions.VmBuildingException,
+            exc_list = set(type(c.exception) for c in err._causes)
+            self.assertEqual({cue_exceptions.VmBuildingException,
                               cue_exceptions.VmBuildingException,
-                              cue_exceptions.VmBuildingException],
+                              cue_exceptions.VmBuildingException},
                              exc_list)
         except Exception as e:
             self.assertEqual(taskflow_exc.WrappedFailure, type(e))
@@ -469,4 +469,4 @@ class CreateClusterTests(base.FunctionalTestCase):
     def tearDown(self):
         for vm_id in self.new_vm_list:
             self.nova_client.servers.delete(vm_id)
-        super(CreateClusterTests, self).tearDown()
+        super(CreateClusterTests, self).tearDown()

cue/tests/unit/api/controllers/v1/test_wsme_objects.py

@@ -51,8 +51,8 @@ class TestClusterObject(base.UnitTestCase):
             "name": "test_cluster",
             "size": "3",
         }
-        self.assertRaises(wsme_exception.InvalidInput, api_cluster.Cluster,
-                          **cluster_value)
+        self.assertRaises((wsme_exception.InvalidInput, TypeError),
+                          api_cluster.Cluster, **cluster_value)
 
     def test_api_cluster_invalid_type_size(self):
         """test cluster object with invalid type 'size'."""
@@ -62,8 +62,8 @@ class TestClusterObject(base.UnitTestCase):
             "name": "test_cluster",
             "size": "three",
         }
-        self.assertRaises(wsme_exception.InvalidInput, api_cluster.Cluster,
-                          **cluster_value)
+        self.assertRaises((wsme_exception.InvalidInput, TypeError),
+                          api_cluster.Cluster, **cluster_value)
 
     def test_api_cluster_invalid_type_volume_size(self):
         """test cluster object with invalid type 'volume_size'."""
@@ -74,8 +74,8 @@ class TestClusterObject(base.UnitTestCase):
             "size": 3,
             "volume_size": "zero"
         }
-        self.assertRaises(wsme_exception.InvalidInput, api_cluster.Cluster,
-                          **cluster_value)
+        self.assertRaises((wsme_exception.InvalidInput, TypeError),
+                          api_cluster.Cluster, **cluster_value)
 
     def test_api_cluster_missing_size(self):
         """test cluster object with missing mandatory field 'size'."""
@@ -163,4 +163,4 @@ class TestEndpointObject(base.UnitTestCase):
         self.assertEqual(endpoint_value['uri'], endpoint.uri)
 
     def tearDown(self):
-        super(TestEndpointObject, self).tearDown()
+        super(TestEndpointObject, self).tearDown()

cue/tests/unit/cmd/test_monitor.py

@@ -34,7 +34,7 @@ class TestMonitor(base.UnitTestCase):
         monitor.main()
 
         mock_tooz_get_coordinator.assert_called_once_with(
-            'zookeeper://localhost:2181', 'cue-monitor')
+            'zookeeper://localhost:2181', b'cue-monitor')
         mock_oslo_service_wait.assert_called_once_with()
         self.assertEqual(mock_conf.call_count, 1)
         self.assertEqual(mock_prepare.call_count, 1)

os_tasklib/common/reduce_task.py

@@ -14,6 +14,7 @@
 # under the License.
 
 import oslo_utils.reflection as reflection
+from six import moves
 import taskflow.task as task
 
 
@@ -55,4 +56,4 @@ class Reduce(task.Task):
 
     def execute(self, *args, **kwargs):
         l = [kwargs[r] for r in self.requires]
-        return reduce(self.functor, l)
+        return moves.reduce(self.functor, l)