Corrected max secs for concurrent trans retries

This was most likely meant as a max 2s delay here, not a max 2ms
delay.

Also includes a related change: when retries for metadata updates are
attempted, make sure we do not have a stale value of the atomic_key
(otherwise we'll just inevitably hit the ConcurrentTransaction issue).

 Conflicts:
	heat/engine/service_software_config.py
	heat/objects/resource.py

Co-Authored-By: Crag Wolfe <cwolfe@redhat.com>
Partial-Bug: #1651768
Change-Id: Ie56e0e4ff93633db1f4752859d2b2a9506922911
(cherry picked from commit e37d9fab8f)

heat/engine/service_software_config.py

@@ -101,6 +101,7 @@ class SoftwareConfigService(service.Service):
         rs = db_api.resource_get_by_physical_resource_id(cnxt, server_id)
         if not rs:
             return
+        cnxt.session.refresh(rs)
         deployments = self.metadata_software_deployments(cnxt, server_id)
         md = rs.rsrc_metadata or {}
         md['deployments'] = deployments

heat/objects/resource.py

@@ -38,8 +38,9 @@ cfg.CONF.import_opt('encrypt_parameters_and_properties', 'heat.common.config')
 def retry_on_conflict(func):
     def is_conflict(ex):
         return isinstance(ex, exception.ConcurrentTransaction)
+
     wrapper = retrying.retry(stop_max_attempt_number=11,
-                             wait_random_min=0.0, wait_random_max=2.0,
+                             wait_random_min=0, wait_random_max=2000,
                              retry_on_exception=is_conflict)
     return wrapper(func)
 

heat/tests/engine/service/test_software_config.py

@@ -635,8 +635,9 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
             'deployments': {'deploy': 'this'}
         }
 
-        self.engine.software_config._push_metadata_software_deployments(
-            self.ctx, '1234', None)
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            self.engine.software_config._push_metadata_software_deployments(
+                self.ctx, '1234', None)
         res_upd.assert_called_once_with(
             self.ctx, '1234', {'rsrc_metadata': result_metadata}, 1)
         put.side_effect = Exception('Unexpected requests.put')
@@ -660,12 +661,14 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
         deployments = {'deploy': 'this'}
         md_sd.return_value = deployments
 
-        self.assertRaises(
-            exception.ConcurrentTransaction,
-            self.engine.software_config._push_metadata_software_deployments,
-            self.ctx,
-            '1234',
-            None)
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            f = self.engine.software_config._push_metadata_software_deployments
+            self.assertRaises(
+                exception.ConcurrentTransaction,
+                f,
+                self.ctx,
+                '1234',
+                None)
         # retry ten times then the final failure
         self.assertEqual(11, res_upd.call_count)
         put.assert_not_called()
@@ -695,9 +698,9 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
             'original': 'metadata',
             'deployments': {'deploy': 'this'}
         }
-
-        self.engine.software_config._push_metadata_software_deployments(
-            self.ctx, '1234', None)
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            self.engine.software_config._push_metadata_software_deployments(
+                self.ctx, '1234', None)
         res_upd.assert_called_once_with(
             self.ctx, '1234', {'rsrc_metadata': result_metadata}, 1)
 
@@ -734,8 +737,9 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
             'deployments': {'deploy': 'this'}
         }
 
-        self.engine.software_config._push_metadata_software_deployments(
-            self.ctx, '1234', 'project1')
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            self.engine.software_config._push_metadata_software_deployments(
+                self.ctx, '1234', 'project1')
         res_upd.assert_called_once_with(
             self.ctx, '1234', {'rsrc_metadata': result_metadata}, 1)
 
@@ -903,8 +907,9 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
         zaqar_client.queue.return_value = queue
         queue.pop.return_value = [mock.Mock(body='ok')]
 
-        deployment = self._create_software_deployment(
-            status='IN_PROGRESS', config_id=config['id'])
+        with mock.patch.object(self.ctx.session, 'refresh'):
+            deployment = self._create_software_deployment(
+                status='IN_PROGRESS', config_id=config['id'])
 
         deployment_id = deployment['id']
         self.assertEqual(