Add stochastic weight handler to scheduler

Add a config option to the scheduler host manager to select host
weight managers other than the default. Rename the existing default
weight manager TopHostWeightHandler because its behavior is to
always choose the top weight. Add a new weight manager which
randomly chooses the host using weights as probabilities.

DocImpact

Implements blueprint stochastic-weighing-scheduler

Change-Id: Ie5985066e68aef25600f8b76948adacd4c04263a

cinder/scheduler/host_manager.py

@@ -21,6 +21,7 @@ import collections
 
 from oslo_config import cfg
 from oslo_log import log as logging
+from oslo_utils import importutils
 from oslo_utils import timeutils
 
 from cinder import context as cinder_context
@@ -29,7 +30,6 @@ from cinder import objects
 from cinder import utils
 from cinder.i18n import _LI, _LW
 from cinder.scheduler import filters
-from cinder.scheduler import weights
 from cinder.volume import utils as vol_utils
 
 
@@ -46,7 +46,11 @@ host_manager_opts = [
                 default=[
                     'CapacityWeigher'
                 ],
-                help='Which weigher class names to use for weighing hosts.')
+                help='Which weigher class names to use for weighing hosts.'),
+    cfg.StrOpt('scheduler_weight_handler',
+               default='cinder.scheduler.weights.OrderedHostWeightHandler',
+               help='Which handler to use for selecting the host/pool '
+                    'after weighing'),
 ]
 
 CONF = cfg.CONF
@@ -347,8 +351,9 @@ class HostManager(object):
         self.filter_handler = filters.HostFilterHandler('cinder.scheduler.'
                                                         'filters')
         self.filter_classes = self.filter_handler.get_all_classes()
-        self.weight_handler = weights.HostWeightHandler('cinder.scheduler.'
-                                                        'weights')
+        self.weight_handler = importutils.import_object(
+            CONF.scheduler_weight_handler,
+            'cinder.scheduler.weights')
         self.weight_classes = self.weight_handler.get_all_classes()
 
         self._no_capabilities_hosts = set()  # Hosts having no capabilities

cinder/scheduler/weights/__init__.py

@@ -37,8 +37,9 @@ class BaseHostWeigher(base_weight.BaseWeigher):
     pass
 
 
-class HostWeightHandler(base_weight.BaseWeightHandler):
+class OrderedHostWeightHandler(base_weight.BaseWeightHandler):
     object_class = WeighedHost
 
     def __init__(self, namespace):
-        super(HostWeightHandler, self).__init__(BaseHostWeigher, namespace)
+        super(OrderedHostWeightHandler, self).__init__(BaseHostWeigher,
+                                                       namespace)

cinder/scheduler/weights/stochastic.py

@@ -0,0 +1,83 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Stochastic weight handler
+
+This weight handler differs from the default weight
+handler by giving every pool a chance to be chosed
+wheret the probability is proportional to each pools'
+weight.
+"""
+
+import random
+
+from cinder.scheduler.base_weight import BaseWeightHandler
+from cinder.scheduler.weights import BaseHostWeigher
+from cinder.scheduler.weights import WeighedHost
+
+
+class StochasticHostWeightHandler(BaseWeightHandler):
+    def __init__(self, namespace):
+        super(StochasticHostWeightHandler, self).__init__(BaseHostWeigher,
+                                                          namespace)
+
+    def get_weighed_objects(self, weigher_classes, obj_list,
+                            weighing_properties):
+        # The normalization performed in the superclass is nonlinear, which
+        # messes up the probabilities, so override it. The probabilistic
+        # approach we use here is self-normalizing.
+        # Also, the sorting done by the parent implementation is harmless but
+        # useless for us.
+
+        # Compute the object weights as the parent would but without sorting
+        # or normalization.
+        weighed_objs = [WeighedHost(obj, 0.0) for obj in obj_list]
+        for weigher_cls in weigher_classes:
+            weigher = weigher_cls()
+            weights = weigher.weigh_objects(weighed_objs, weighing_properties)
+            for i, weight in enumerate(weights):
+                obj = weighed_objs[i]
+                obj.weight += weigher.weight_multiplier() * weight
+
+        # Avoid processing empty lists
+        if not weighed_objs:
+            return []
+
+        # First compute the total weight of all the objects and the upper
+        # bound for each object to "win" the lottery.
+        total_weight = 0
+        table = []
+        for weighed_obj in weighed_objs:
+            total_weight += weighed_obj.weight
+            max_value = total_weight
+            table.append((max_value, weighed_obj))
+
+        # Now draw a random value with the computed range
+        winning_value = random.random() * total_weight
+
+        # Scan the table to find the first object with a maximum higher than
+        # the random number. Save the index of the winner.
+        winning_index = 0
+        for (i, (max_value, weighed_obj)) in enumerate(table):
+            if max_value > winning_value:
+                # Return a single element array with the winner.
+                winning_index = i
+                break
+
+        # It's theoretically possible for the above loop to terminate with no
+        # winner. This happens when winning_value >= total_weight, which
+        # could only occur with very large numbers and floating point
+        # rounding. In those cases the actual winner should have been the
+        # last element, so return it.
+        return weighed_objs[winning_index:] + weighed_objs[0:winning_index]

cinder/tests/unit/scheduler/test_allocated_capacity_weigher.py

@@ -33,7 +33,7 @@ class AllocatedCapacityWeigherTestCase(test.TestCase):
     def setUp(self):
         super(AllocatedCapacityWeigherTestCase, self).setUp()
         self.host_manager = fakes.FakeHostManager()
-        self.weight_handler = weights.HostWeightHandler(
+        self.weight_handler = weights.OrderedHostWeightHandler(
             'cinder.scheduler.weights')
 
     def _get_weighed_host(self, hosts, weight_properties=None):

cinder/tests/unit/scheduler/test_capacity_weigher.py

@@ -32,7 +32,7 @@ class CapacityWeigherTestCase(test.TestCase):
     def setUp(self):
         super(CapacityWeigherTestCase, self).setUp()
         self.host_manager = fakes.FakeHostManager()
-        self.weight_handler = weights.HostWeightHandler(
+        self.weight_handler = weights.OrderedHostWeightHandler(
             'cinder.scheduler.weights')
 
     def _get_weighed_hosts(self, hosts, weight_properties=None):

cinder/tests/unit/scheduler/test_stochastic_weight_handler.py

@@ -0,0 +1,67 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Tests for stochastic weight handler
+"""
+
+import ddt
+import mock
+import random
+
+from cinder.scheduler import base_weight
+from cinder.scheduler.weights.stochastic import StochasticHostWeightHandler
+from cinder import test
+
+
+@ddt.ddt
+class StochasticWeightHandlerTestCase(test.TestCase):
+    """Test case for StochasticHostWeightHandler."""
+
+    def setUp(self):
+        super(StochasticWeightHandlerTestCase, self).setUp()
+
+    @ddt.data(
+        (0.0, 'A'),
+        (0.1, 'A'),
+        (0.2, 'B'),
+        (0.3, 'B'),
+        (0.4, 'B'),
+        (0.5, 'B'),
+        (0.6, 'B'),
+        (0.7, 'C'),
+        (0.8, 'C'),
+        (0.9, 'C'),
+    )
+    @ddt.unpack
+    def test_get_weighed_objects_correct(self, rand_value, expected_obj):
+        self.mock_object(random,
+                         'random',
+                         mock.Mock(return_value=rand_value))
+
+        class MapWeigher(base_weight.BaseWeigher):
+            minval = 0
+            maxval = 100
+
+            def _weigh_object(self, obj, weight_map):
+                return weight_map[obj]
+
+        weight_map = {'A': 1, 'B': 3, 'C': 2}
+        objs = sorted(weight_map.keys())
+
+        weigher_classes = [MapWeigher]
+        handler = StochasticHostWeightHandler('fake_namespace')
+        weighted_objs = handler.get_weighed_objects(weigher_classes,
+                                                    objs,
+                                                    weight_map)
+        winner = weighted_objs[0].obj
+        self.assertEqual(expected_obj, winner)

cinder/tests/unit/scheduler/test_volume_number_weigher.py

@@ -58,7 +58,7 @@ class VolumeNumberWeigherTestCase(test.TestCase):
                                               read_deleted="no",
                                               overwrite=False)
         self.host_manager = fakes.FakeHostManager()
-        self.weight_handler = weights.HostWeightHandler(
+        self.weight_handler = weights.OrderedHostWeightHandler(
             'cinder.scheduler.weights')
 
     def _get_weighed_host(self, hosts, weight_properties=None):

releasenotes/notes/add-stochastic-scheduling-option-99e10eae023fbcca.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - Added a new config option `scheduler_weight_handler`. This is a global
+    option which specifies how the scheduler should choose from a listed of
+    weighted pools. By default the existing weigher is used which always
+    chooses the highest weight.
+  - Added a new weight handler `StochasticHostWeightHandler`. This weight
+    handler chooses pools randomly, where the random probabilities are
+    proportional to the weights, so higher weighted pools are chosen more
+    frequently, but not all the time. This weight handler spreads new
+    shares across available pools more fairly.