Add randomness to physical host selection

In the event that a given physical node has reliability issues, it can
be difficult for users to "get around" the problem because we currently
deterministically pick the first node from the list of nodes available.
Adding randomness to this process helps spread the probability that a
given lease will nab a node that has issues (but has not yet been
identified as such.)

Co-Authored-By: Matt Crees <mattc@stackhpc.com>

Change-Id: I0f0ddb14ed5d21f0ccc5c9659c821d58cea5cbde

blazar/plugins/oshosts/host_plugin.py

@@ -15,6 +15,7 @@
 # under the License.
 
 import datetime
+from random import Random
 
 from novaclient import exceptions as nova_exceptions
 from oslo_config import cfg
@@ -64,6 +65,9 @@ plugin_opts = [
                help='Interval (minutes) of reservation healing. '
                     'If 0 is specified, the interval is infinite and all the '
                     'reservations in the future is healed at one time.'),
+    cfg.BoolOpt('randomize_host_selection',
+                default=False,
+                help='Allocate hosts for reservations randomly.'),
 ]
 
 CONF = cfg.CONF
@@ -593,9 +597,13 @@ class PhysicalHostPlugin(base.BasePlugin, nova.NovaClientWrapper):
             ]:
                 allocated_host_ids.append(host['id'])
         if len(not_allocated_host_ids) >= int(min_host):
+            if CONF[self.resource_type].randomize_host_selection:
+                Random.shuffle(not_allocated_host_ids)
             return not_allocated_host_ids[:int(max_host)]
         all_host_ids = allocated_host_ids + not_allocated_host_ids
         if len(all_host_ids) >= int(min_host):
+            if CONF[self.resource_type].randomize_host_selection:
+                Random.shuffle(all_host_ids)
             return all_host_ids[:int(max_host)]
         else:
             return []

blazar/tests/plugins/oshosts/test_physical_host_plugin.py

@@ -22,6 +22,7 @@ from novaclient import client as nova_client
 from novaclient import exceptions as nova_exceptions
 from oslo_config import cfg
 from oslo_config import fixture as conf_fixture
+import random
 import testtools
 
 from blazar import context
@@ -2333,8 +2334,115 @@ class PhysicalHostPluginTestCase(tests.TestCase):
             '[]', '[]', '3-3',
             datetime.datetime(2013, 12, 19, 20, 00),
             datetime.datetime(2013, 12, 19, 21, 00))
+        self.addCleanup(CONF.clear_override, 'cleaning_time')
         self.assertEqual(['host1', 'host2', 'host3'], result)
 
+    @mock.patch.object(random.Random, "shuffle")
+    def test_random_matching_hosts_not_allocated_hosts(self, mock_shuffle):
+        def host_allocation_get_all_by_values(**kwargs):
+            if kwargs['compute_host_id'] == 'host1':
+                return True
+        self.cfg.CONF.set_override('randomize_host_selection', True,
+                                   group=plugin.RESOURCE_TYPE)
+        host_get = self.patch(
+            self.db_api,
+            'reservable_host_get_all_by_queries')
+        host_get.return_value = [
+            {'id': 'host1'},
+            {'id': 'host2'},
+            {'id': 'host3'},
+        ]
+        host_get = self.patch(
+            self.db_api,
+            'host_allocation_get_all_by_values')
+        host_get.side_effect = host_allocation_get_all_by_values
+        host_get = self.patch(
+            self.db_utils,
+            'get_free_periods')
+        host_get.return_value = [
+            (datetime.datetime(2013, 12, 19, 20, 00),
+             datetime.datetime(2013, 12, 19, 21, 00)),
+        ]
+        self.fake_phys_plugin._matching_hosts(
+            '[]', '[]', '1-3',
+            datetime.datetime(2013, 12, 19, 20, 00),
+            datetime.datetime(2013, 12, 19, 21, 00))
+        self.addCleanup(CONF.clear_override, 'randomize_host_selection',
+                        group=plugin.RESOURCE_TYPE)
+        mock_shuffle.assert_called_once_with(['host2', 'host3'])
+
+    @mock.patch.object(random.Random, "shuffle")
+    def test_random_matching_hosts_allocated_hosts(self, mock_shuffle):
+        def host_allocation_get_all_by_values(**kwargs):
+            if kwargs['compute_host_id'] == 'host1':
+                return True
+        self.cfg.CONF.set_override('randomize_host_selection', True,
+                                   group=plugin.RESOURCE_TYPE)
+        host_get = self.patch(
+            self.db_api,
+            'reservable_host_get_all_by_queries')
+        host_get.return_value = [
+            {'id': 'host1'},
+            {'id': 'host2'},
+            {'id': 'host3'},
+        ]
+        host_get = self.patch(
+            self.db_api,
+            'host_allocation_get_all_by_values')
+        host_get.side_effect = host_allocation_get_all_by_values
+        host_get = self.patch(
+            self.db_utils,
+            'get_free_periods')
+        host_get.return_value = [
+            (datetime.datetime(2013, 12, 19, 20, 00),
+             datetime.datetime(2013, 12, 19, 21, 00)),
+        ]
+        self.fake_phys_plugin._matching_hosts(
+            '[]', '[]', '3-3',
+            datetime.datetime(2013, 12, 19, 20, 00),
+            datetime.datetime(2013, 12, 19, 21, 00))
+        self.addCleanup(CONF.clear_override, 'randomize_host_selection',
+                        group=plugin.RESOURCE_TYPE)
+        mock_shuffle.assert_called_once_with(['host1', 'host2', 'host3'])
+
+    @mock.patch.object(random.Random, "shuffle")
+    def test_random_matching_hosts_allocated_cleaning_time(self, mock_shuffle):
+        def host_allocation_get_all_by_values(**kwargs):
+            if kwargs['compute_host_id'] == 'host1':
+                return True
+        self.cfg.CONF.set_override('randomize_host_selection', True,
+                                   group=plugin.RESOURCE_TYPE)
+        self.cfg.CONF.set_override('cleaning_time', '5')
+        host_get = self.patch(
+            self.db_api,
+            'reservable_host_get_all_by_queries')
+        host_get.return_value = [
+            {'id': 'host1'},
+            {'id': 'host2'},
+            {'id': 'host3'},
+        ]
+        host_get = self.patch(
+            self.db_api,
+            'host_allocation_get_all_by_values')
+        host_get.side_effect = host_allocation_get_all_by_values
+        host_get = self.patch(
+            self.db_utils,
+            'get_free_periods')
+        host_get.return_value = [
+            (datetime.datetime(2013, 12, 19, 20, 00)
+             - datetime.timedelta(minutes=5),
+             datetime.datetime(2013, 12, 19, 21, 00)
+             + datetime.timedelta(minutes=5))
+        ]
+        self.fake_phys_plugin._matching_hosts(
+            '[]', '[]', '3-3',
+            datetime.datetime(2013, 12, 19, 20, 00),
+            datetime.datetime(2013, 12, 19, 21, 00))
+        self.addCleanup(CONF.clear_override, 'randomize_host_selection',
+                        group=plugin.RESOURCE_TYPE)
+        self.addCleanup(CONF.clear_override, 'cleaning_time')
+        mock_shuffle.assert_called_once_with(['host1', 'host2', 'host3'])
+
     def test_matching_hosts_not_matching(self):
         host_get = self.patch(
             self.db_api,

releasenotes/notes/option-for-randomising-host-selection-101e95baab67dd88.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds optional randomness to physical host allocation. This is disabled by
+    default and can be enabled with the configuration option
+    ``[physical:host]/randomize_host_selection``.