From b1cef8c4a0470ce978814ded0ce707b403d839b1 Mon Sep 17 00:00:00 2001 From: Evgeniy L Date: Wed, 30 Mar 2016 15:06:51 +0300 Subject: [PATCH] Refactoring of allocation solvers. Implemented layering so it will allow to have multiple solver engines. Implements blueprint: dynamic-allocation Change-Id: I7ed1ec0216fb9778b4fa5be4fb4f6141a0e26fc9 --- bareon_allocator/allocators.py | 393 +----------------- bareon_allocator/objects/space.py | 6 +- .../parsers/dynamic_schema_parser.py | 6 +- bareon_allocator/solvers/__init__.py | 24 ++ bareon_allocator/solvers/base.py | 38 ++ bareon_allocator/solvers/linear_program.py | 97 +++++ .../solvers/linear_program_creator.py | 331 +++++++++++++++ .../linear_programming_scipy_solver.py | 108 +++++ bareon_allocator/solvers/utils.py | 26 ++ .../tests/test_bareon_dynamic_allocator.py | 28 -- bareon_allocator/tests/test_objects_disk.py | 26 ++ bareon_allocator/tests/test_objects_space.py | 44 ++ .../test_parsers_dynamic_schema_parser.py | 62 +++ .../tests/test_parsers_expressions.py | 33 ++ .../tests/test_solvers_linear_program.py | 67 +++ .../test_solvers_linear_program_creator.py | 173 ++++++++ ...solvers_linear_programming_scipy_solver.py | 55 +++ bareon_allocator/tests/test_solvers_utils.py | 27 ++ doc/source/architecture.rst | 2 +- 19 files changed, 1137 insertions(+), 409 deletions(-) create mode 100644 bareon_allocator/solvers/__init__.py create mode 100644 bareon_allocator/solvers/base.py create mode 100644 bareon_allocator/solvers/linear_program.py create mode 100644 bareon_allocator/solvers/linear_program_creator.py create mode 100644 bareon_allocator/solvers/linear_programming_scipy_solver.py create mode 100644 bareon_allocator/solvers/utils.py delete mode 100644 bareon_allocator/tests/test_bareon_dynamic_allocator.py create mode 100644 bareon_allocator/tests/test_objects_disk.py create mode 100644 bareon_allocator/tests/test_objects_space.py create mode 100644 bareon_allocator/tests/test_parsers_dynamic_schema_parser.py create mode 100644 bareon_allocator/tests/test_parsers_expressions.py create mode 100644 bareon_allocator/tests/test_solvers_linear_program.py create mode 100644 bareon_allocator/tests/test_solvers_linear_program_creator.py create mode 100644 bareon_allocator/tests/test_solvers_linear_programming_scipy_solver.py create mode 100644 bareon_allocator/tests/test_solvers_utils.py diff --git a/bareon_allocator/allocators.py b/bareon_allocator/allocators.py index def4502..ec2af50 100644 --- a/bareon_allocator/allocators.py +++ b/bareon_allocator/allocators.py @@ -14,21 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. -import itertools -import math - -import numpy as np - from oslo_log import log -from scipy.optimize import linprog -from bareon_allocator import errors from bareon_allocator.parsers import DynamicSchemaParser +from bareon_allocator.solvers import LinearProgramCreator +from bareon_allocator.solvers import LinearProgrammingScipySolver from bareon_allocator import utils -from bareon_allocator.sequences import CrossSumInequalitySequence - - LOG = log.getLogger(__name__) @@ -37,388 +29,39 @@ class DynamicAllocator(object): def __init__(self, hw_info, schema): LOG.debug('Hardware information: %s', hw_info) LOG.debug('Spaces schema: %s', schema) - dynamic_schema = DynamicSchemaParser(hw_info, schema) - LOG.debug('Spaces objects: %s', dynamic_schema.spaces) - LOG.debug('Disks objects: \n%s', dynamic_schema.disks) + self.dynamic_schema = DynamicSchemaParser(hw_info, schema) + LOG.debug('Spaces objects: %s', self.dynamic_schema.spaces) + LOG.debug('Disks objects: %s', self.dynamic_schema.disks) - self.solver = DynamicAllocationLinearProgram( - dynamic_schema.disks, - dynamic_schema.spaces) + linear_program = LinearProgramCreator( + self.dynamic_schema).linear_program() + self.solver = LinearProgrammingScipySolver(linear_program) def generate_static(self): - sizes = self.solver.solve() - - return sizes - - -class DynamicAllocationLinearProgram(object): - """Linear programming allocator. - - Use Linear Programming method [0] (the method itself has nothing to do - with computer-programming) in order to formulate and solve the problem - of spaces allocation on disks, with the best outcome. - - In this implementation scipy is being used since it already implements - simplex algorithm to find the best feasible solution. - - [0] https://en.wikipedia.org/wiki/Linear_programming - [1] http://docs.scipy.org/doc/scipy-0.16.0/reference/generated - /scipy.optimize.linprog.html - [2] https://en.wikipedia.org/wiki/Simplex_algorithm - """ - - weight_set_mapping = [ - # Don't use minimal size, in this case - # we will get a weight for the space which - # in combination with space which has max_size - # so there will be unallocated space - # ['min_size', 'best_with_disks'], - # ['max_size', 'best_with_disks'], - ['min_size', 'max_size', 'best_with_disks']] - - def __init__(self, disks, spaces): - self.disks = disks - self.spaces = spaces - # Coefficients of the linear objective minimization function. - # During iteration over vertexes the function is used to identify - # if current solution (vertex) satisfies the equation more, than - # previous one. - # Example of equation: c[0]*x1 + c[1]*x2 - self.objective_function_coefficients = [] - - # A matrix which, gives the values of the equality constraints at x, - # when multipled by x. - self.equality_constraint_matrix = [] - - # An array of values representing right side of equation, - # left side is represented by row of `equality_constraint_matrix` - # matrix - self.equality_constraint_vector = np.array([]) - - self.upper_bound_constraint_matrix = [] - self.upper_bound_constraint_vector = [] - self.lower_bound_constraint_matrix = [] - self.lower_bound_constraint_vector = [] - - # Specify boundaries of each x in the next format (min, max). Use - # None for one of min or max when there is no bound. - self.bounds = np.array([]) - - # For each space, xn (size of the space) is represented - # for each disk as separate variable, so for each - # disk we have len(spaces) * len(disks) sizes - self.x_amount = len(self.disks) * len(self.spaces) - - # TODO(eli): has to be refactored - # Here we store indexes for bounds and equation - # matrix, in order to be able to change it on - # refresh - self.weight_equation_indexes = [] - - self._set_spaces_sets_by(self.weight_set_mapping[0]) - self._init_equation(self.disks, self.spaces) - self._init_objective_function_coefficient() - self._init_min_max() - self._refresh_weight() - - def solve(self): - upper_bound_matrix = self._make_upper_bound_constraint_matrix() or None - upper_bound_vector = self._make_upper_bound_constraint_vector() or None - - LOG.debug('Objective function coefficients human-readable:\n%s\n', - utils.format_x_vector(self.objective_function_coefficients, - len(self.spaces))) - - LOG.debug('Equality equation:\n%s\n', - utils.format_equation( - self.equality_constraint_matrix, - self.equality_constraint_vector, - len(self.spaces))) - LOG.debug('Inequality equation:\n%s\n', - utils.format_equation( - upper_bound_matrix, - upper_bound_vector, - len(self.spaces))) - - for weight_for_sets in self.weight_set_mapping: - LOG.debug('Parameters for spaces set formation: %s', - weight_for_sets) - self._set_spaces_sets_by(weight_for_sets) - solution = linprog( - self.objective_function_coefficients, - A_eq=self.equality_constraint_matrix, - b_eq=self.equality_constraint_vector, - A_ub=upper_bound_matrix, - b_ub=upper_bound_vector, - bounds=self.bounds, - options={"disp": False}) - - # If solution is found we can finish attempts to find - # the best solution - if not solution.success: - break - - LOG.debug("Solution: %s", solution) - self._check_errors(solution) - # Naive implementation of getting integer result - # from a linear programming algorithm, MIP - # (mixed integer programming) should be considered - # instead, but it may have a lot of problems (solution - # of such equations is NP-hard in some cases), - # for our practical purposes it's enough to round - # the number down, in this case we may get `n` megabytes - # unallocated, where n is len(spaces) * len(disks) - solution_vector = self._round_down(solution.x) - - return self._convert_solution(solution_vector) - - def _check_errors(self, solution): - if not solution.success: - raise errors.NoSolutionFound( - 'Allocation is not possible ' - 'with specified constraints: {0}'.format(solution.message)) - - def _round_down(self, vector): - return [int(math.floor(f)) for f in vector] - - def _init_min_max(self): - """Create min and max constraints for each space. - - In case of 2 disks and 2 spaces - - For first space min_size >= 10 and max_size <= 20 - 1 * x1 + 0 * x2 + 1 * x3 + 0 * x4 >= 10 - 1 * x1 + 0 * x2 + 1 * x3 + 0 * x4 <= 20 - - For second space min_size >= 15 and max_size <= 30 - 0 * x1 + 1 * x2 + 0 * x3 + 1 * x4 >= 15 - 0 * x1 + 1 * x2 + 0 * x3 + 1 * x4 <= 30 - """ - for space_idx, space in enumerate(self.spaces): - row = self._make_matrix_row() - max_size = getattr(space, 'max_size', None) - min_size = getattr(space, 'min_size', None) - - for disk_idx in range(len(self.disks)): - row[disk_idx * len(self.spaces) + space_idx] = 1 - - if min_size is not None: - self.lower_bound_constraint_matrix.append(row) - self.lower_bound_constraint_vector.append(min_size) - - if max_size is not None: - self.upper_bound_constraint_matrix.append(row) - self.upper_bound_constraint_vector.append(max_size) - - def _get_spaces_sets_by(self, criteria): - return [i[1] for i in self._get_sets_by(criteria)] - - def _get_sets_by(self, criteria): - def get_values(space): - return [getattr(space, c, None) for c in criteria] - - grouped_spaces = itertools.groupby( - sorted(self.spaces, key=get_values), - key=get_values) - - return [(k, list(v)) for k, v in grouped_spaces] - - def _set_spaces_sets_by(self, criteria): - self.weight_spaces_sets = self._get_spaces_sets_by(criteria) - - def _refresh_weight(self): - """Refresh weight. - - Create weight constraints for spaces which have same - max constraint or for those which don't have it at all. - - Lets say, second's space is equal to max of the third and fourth, - we will have next equation: - 0 * x1 + (1 / weight) * x2 + (-1 / weight) * x3 + - 0 * x4 + (1 / weight) * x5 + (-1 / weight) * x6 = 0 - """ - DEFAULT_WEIGHT = 1 - # Clean constraint matrix and vector from previous values - for idx in sorted(self.weight_equation_indexes, reverse=True): - del self.equality_constraint_matrix[idx] - del self.equality_constraint_vector[idx] - self.weight_equation_indexes = [] - - for spaces_set in self.weight_spaces_sets: - # Don't set weight if there is less than one space in the set - if len(spaces_set) < 2: - continue - - first_weight = getattr(spaces_set[0], 'weight', DEFAULT_WEIGHT) - first_space_idx = self.spaces.index(spaces_set[0]) - for space in spaces_set[1:]: - row = self._make_matrix_row() - weight = getattr(space, 'weight', DEFAULT_WEIGHT) - - # If weight is 0, it doesn't make sense to set for such - # space a weight - if weight == 0: - continue - - space_idx = self.spaces.index(space) - - for disk_idx in range(len(self.disks)): - row_i = disk_idx * len(self.spaces) - row[row_i + first_space_idx] = 1 / first_weight - row[row_i + space_idx] = -1 / weight - - self.weight_equation_indexes.append( - len(self.equality_constraint_matrix) - 1) - - self.equality_constraint_matrix.append(row) - self.equality_constraint_vector = np.append( - self.equality_constraint_vector, - 0) - - def _make_matrix_row(self): - return np.zeros(self.x_amount) - - def _make_upper_bound_constraint_matrix(self): - """Creates upper bound constraint matrix. - - Upper bound constraint matrix consist of upper bound - matrix and lower bound matrix witch changed sign. - """ - return (self.upper_bound_constraint_matrix + - [[-i for i in row] - for row in self.lower_bound_constraint_matrix]) - - def _make_upper_bound_constraint_vector(self): - """Create upper bound constraint vector. - - Upper bound constraint vector consist of upper bound and - lower bound, with changed sign. - """ - return (self.upper_bound_constraint_vector + - [-i for i in self.lower_bound_constraint_vector]) + solution = self.solver.solve() + LOG.debug('Static allocation schema: \n%s', solution) + return self._convert_solution(solution) def _convert_solution(self, solution_vector): + # TODO(eli): convertation logic should be moved to solvers, + # as result Solver object should be returned and used result = [] spaces_grouped_by_disk = list(utils.grouper( solution_vector, - len(self.spaces))) - for disk_i in range(len(self.disks)): - disk_id = self.disks[disk_i].id + len(self.dynamic_schema.spaces))) + for disk_i in range(len(self.dynamic_schema.disks)): + disk_id = self.dynamic_schema.disks[disk_i].id disk = {'disk_id': disk_id, - 'size': self.disks[disk_i].size, + 'size': self.dynamic_schema.disks[disk_i].size, 'spaces': []} spaces_for_disk = spaces_grouped_by_disk[disk_i] for space_i, space_size in enumerate(spaces_for_disk): disk['spaces'].append({ - 'space_id': self.spaces[space_i].id, + 'space_id': self.dynamic_schema.spaces[space_i].id, 'size': space_size}) result.append(disk) return result - - def _init_equation(self, disks, spaces): - for d in disks: - # Initialize constraints, each row in the matrix should - # be equal to size of the disk - self.equality_constraint_vector = np.append( - self.equality_constraint_vector, - d.size) - - # Initialize the matrix - # In case of 2 spaces and 3 disks the result should be: - # [[1, 1, 0, 0, 0, 0], - # [0, 0, 1, 1, 0, 0], - # [0, 0, 0, 0, 1, 1]] - # - # Explanation of the first row - # [1, - x1 multiplier, size of space 1 on the first disk - # 1, - x2 multiplier, size of space 2 on the first disk - # 0, - x3 multiplier, size of space 1 on 2nd disk, 0 for the first - # 0, - x4 multiplier, size of space 2 on 2nd disk, 0 for the first - # 0, - x5 multiplier, size of space 1 on 3rd disk, 0 for the first - # 0] - x6 multiplier, size of space 2 on 3rd disk, 0 for the first - equality_matrix_row = self._make_matrix_row() - - # Set first len(spaces) elements to 1 - equality_matrix_row = utils.shift( - equality_matrix_row, - len(spaces), - val=1) - - for _ in range(len(disks)): - self.equality_constraint_matrix.append(equality_matrix_row) - equality_matrix_row = utils.shift( - equality_matrix_row, - len(spaces), - val=0) - - # Size of each space should be more or equal to 0 - for _ in range(self.x_amount): - self._add_bound(0, None) - - def _init_objective_function_coefficient(self): - # Amount of coefficients is equal to amount of x - c_amount = self.x_amount - - # We want spaces to be allocated on disks - # in order which user specified them in the schema. - # In order to do that, we set coefficients - # higher for those spaces which defined earlier - # in the list - - # TODO(eli): describe why we should use special sequence - # as order coefficients - coefficients = [1.0 / i for i in CrossSumInequalitySequence(c_amount)] - - NONE_ORDER_COEFF = 1 - SET_COEFF = 2 - - space_sets = self._get_spaces_sets_by(['best_with_disks']) - - # A list of disks ids which are not selected for specific spaces - all_disks_ids = [i for i in range(len(self.disks))] - used_disks_ids = [] - - for k, space in self._get_sets_by(['best_with_disks']): - if k[0]: - used_disks_ids.extend(list(k[0])) - - not_best_disks = list(set(all_disks_ids) - set(used_disks_ids)) - - for i_set, space_set in enumerate(space_sets): - for space in space_set: - s_i = self.spaces.index(space) - - for d_i in range(len(self.disks)): - c_i = len(self.spaces) * d_i + s_i - - # Set constant for none_order spaces - if getattr(space, 'none_order', False): - coefficients[c_i] = NONE_ORDER_COEFF - continue - - if space.best_with_disks: - if d_i in space.best_with_disks: - coefficients[c_i] += SET_COEFF - else: - # If current disk is not in the set, set it to 0 - # TODO(eli): isn't it better to leave there order - # coefficient? - # coefficients[c_i] = 0 - pass - else: - # Don't allcoate coefficient for the spaces - # which have no best_with_disks, on best_with_disks - if d_i in not_best_disks: - coefficients[c_i] += SET_COEFF - - # By default the algorithm tries to minimize the solution - # we should invert sign, in order to make it a maximization - # function, because we want disks to be maximally allocated. - self.objective_function_coefficients = [-c for c in coefficients] - - def _add_bound(self, min_, max_): - np.append(self.bounds, (min_, max_)) diff --git a/bareon_allocator/objects/space.py b/bareon_allocator/objects/space.py index 898890e..d5bf295 100644 --- a/bareon_allocator/objects/space.py +++ b/bareon_allocator/objects/space.py @@ -24,14 +24,16 @@ class Space(BaseObject): 'min_size': 0, 'max_size': None, 'best_with_disks': set([]), - 'weight': 1 + 'weight': 1, + 'none_order': False, + 'type': None } required = ['id', 'type'] def __init__(self, **kwargs): super(Space, self).__init__(**kwargs) - # Exact size should be repreneted as min_size and max_size + # Exact size should be represented as min_size and max_size if kwargs.get('size'): self.min_size = kwargs.get('size') self.max_size = kwargs.get('size') diff --git a/bareon_allocator/parsers/dynamic_schema_parser.py b/bareon_allocator/parsers/dynamic_schema_parser.py index a1819c4..5e6c1f0 100644 --- a/bareon_allocator/parsers/dynamic_schema_parser.py +++ b/bareon_allocator/parsers/dynamic_schema_parser.py @@ -70,13 +70,13 @@ class DynamicSchemaParser(object): for i, space in enumerate(spaces): if space.get('best_with_disks'): - disks_idx = set() + disks_ids = set() for disk in space['best_with_disks']: try: - disks_idx.add(self.raw_disks.index(disk)) + disks_ids.add(disk['id']) except ValueError as exc: LOG.warn('Warning: %s', exc) - spaces[i]['best_with_disks'] = disks_idx + spaces[i]['best_with_disks'] = disks_ids return spaces diff --git a/bareon_allocator/solvers/__init__.py b/bareon_allocator/solvers/__init__.py new file mode 100644 index 0000000..aab52b2 --- /dev/null +++ b/bareon_allocator/solvers/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# flake8: noqa + +from bareon_allocator.solvers.base import BaseSolver +from bareon_allocator.solvers.linear_program import LinearProgram +from bareon_allocator.solvers.linear_programming_scipy_solver \ + import LinearProgrammingScipySolver +from bareon_allocator.solvers.linear_program_creator \ + import LinearProgramCreator diff --git a/bareon_allocator/solvers/base.py b/bareon_allocator/solvers/base.py new file mode 100644 index 0000000..4767fb3 --- /dev/null +++ b/bareon_allocator/solvers/base.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc + +import six + + +@six.add_metaclass(abc.ABCMeta) +class BaseSolver(object): + """Base class for Bareon Allocator Objects.""" + + def __init__(self, linear_program): + """Initialize object. + + :param linear_program: `class`:LinearProgram object + """ + self.linear_program = linear_program + + @abc.abstractmethod + def solve(self): + """Returns solution hash. + + :raises: errors.NoSolutionFound + """ diff --git a/bareon_allocator/solvers/linear_program.py b/bareon_allocator/solvers/linear_program.py new file mode 100644 index 0000000..cf4c55a --- /dev/null +++ b/bareon_allocator/solvers/linear_program.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class LinearProgram(object): + """LinearProgram object is abstract way to describe linear program.""" + MAXIMIZE = 'maximize' + MINIMIZE = 'minimize' + + # Linear Program + LP_TYPE_LP = 'lp' + # Mixed Integer Program + LP_TYPE_MIP = 'mip' + + def __init__( + self, + x_amount=0, + optimization_type=MAXIMIZE, + lp_type=LP_TYPE_LP, + objective_function_coefficients=None, + + equality_constraint_matrix=None, + lower_constraint_matrix=None, + upper_constraint_matrix=None, + + equality_constraint_vector=None, + lower_constraint_vector=None, + upper_constraint_vector=None): + + self.lp_type = lp_type + self.objective_function_optimization_type = optimization_type + + # Coefficients of the linear objective minimization function. + # During iteration over vertexes the function is used to identify + # if current solution (vertex) satisfies the equation more, than + # previous one. + # Example of equation: c[0]*x1 + c[1]*x2 + self.objective_function_coefficients = objective_function_coefficients + + # Matrices which, gives values of the equality/inequality + # constraints, when multiplied by x. + self.equality_constraint_matrix = equality_constraint_matrix + self.lower_constraint_matrix = lower_constraint_matrix + self.upper_constraint_matrix = upper_constraint_matrix + + # Vectors in combination with equality matrices give + # equality/inequality system of linear equations. + self.equality_constraint_vector = equality_constraint_vector + self.lower_constraint_vector = lower_constraint_vector + self.upper_constraint_vector = upper_constraint_vector + + # Amount unknown of variables. + self.x_amount = x_amount + + # A list of tuples which represents min and max possible values for + # each variable. + self.bounds = [(0, None) for _ in xrange(self.x_amount)] + + def minimize_objective_function(self): + """Minimize objective function.""" + self.objective_function_optimization_type = self.MINIMIZE + + def maximize_objective_function(self): + """Maximize objective function.""" + self.objective_function_optimization_type = self.MAXIMIZE + + def set_type_lp(self): + """Set type of linear program to Linear Program. + + Is default, produces real number result, without any integer + constraints. + """ + self.lp_type = self.LP_TYPE_LP + + def set_type_mip(self): + """"Set type of linear program to Mixed Integer Program. + + This type may include integer constraints, as result wider range of + operations may be available. + + Note: Not all linear programming solvers support this type. + See: https://en.wikipedia.org/wiki/Integer_programming + """ + self.lp_type = self.LP_TYPE_MIP diff --git a/bareon_allocator/solvers/linear_program_creator.py b/bareon_allocator/solvers/linear_program_creator.py new file mode 100644 index 0000000..8a83b73 --- /dev/null +++ b/bareon_allocator/solvers/linear_program_creator.py @@ -0,0 +1,331 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools + +from bareon_allocator.sequences import CrossSumInequalitySequence +from bareon_allocator.solvers.linear_program import LinearProgram + + +class LinearProgramCreator(object): + """Creates LinearProgram based on DynamicSchema object.""" + + NONE_ORDER_COEFFICIENT = 1 + SET_COEFFICIENT = 2 + + def __init__(self, + dynamic_schema, + weight_sets_criteria=[ + 'min_size', + 'max_size', + 'best_with_disks']): + """Initializes the object. + + :param dynamic_schema: :class:`DynamicSchema` object + :param weight_sets_criteria: a list of strings, which represents + attributes of spaces based on which sets will be created to + make equations. + """ + self.weight_sets_criteria = weight_sets_criteria + self.disks = dynamic_schema.disks + self.spaces = dynamic_schema.spaces + + self.spaces_len = len(self.spaces) + self.disks_len = len(self.disks) + + # For each space, x (size of the space) is represented + # for each disk as separate variable, so for each + # disk we have len(spaces) * len(disks) sizes + self.x_amount = self.disks_len * self.spaces_len + + def linear_program(self): + """Returns linear program object + + :return: :class:`LinearProgram` linear program object + """ + space_size_equation = self._make_space_size_constraints() + disk_size_equation = self._make_disk_size_constraints() + equality_weight_equation = self._make_weight_constraints() + + # Merge both equality and constraint vectors into a single dictionary + equations = self._merge_equations(space_size_equation, + disk_size_equation) + equations = self._merge_equations(equations, + equality_weight_equation) + + objective_coefficients = self._make_objective_function_coefficient() + return LinearProgram( + x_amount=self.x_amount, + optimization_type=LinearProgram.MAXIMIZE, + objective_function_coefficients=objective_coefficients, + **equations) + + def _make_space_size_constraints(self): + """Create min and max constraints for each space. + + In case of 2 disks and 2 spaces + + For first space min_size >= 10 and max_size <= 20 + 1 * x1 + 0 * x2 + 1 * x3 + 0 * x4 >= 10 + 1 * x1 + 0 * x2 + 1 * x3 + 0 * x4 <= 20 + + For second space min_size >= 15 and max_size <= 30 + 0 * x1 + 1 * x2 + 0 * x3 + 1 * x4 >= 15 + 0 * x1 + 1 * x2 + 0 * x3 + 1 * x4 <= 30 + """ + constraint_equation = { + 'lower_constraint_matrix': [], + 'lower_constraint_vector': [], + 'upper_constraint_matrix': [], + 'upper_constraint_vector': []} + + for space_idx, space in enumerate(self.spaces): + row = self._make_matrix_row() + + for disk_idx in range(self.disks_len): + row[disk_idx * self.spaces_len + space_idx] = 1 + + if space.min_size is not None: + constraint_equation['lower_constraint_matrix'].append( + row) + constraint_equation['lower_constraint_vector'].append( + space.min_size) + + if space.max_size is not None: + constraint_equation['upper_constraint_matrix'].append( + row) + constraint_equation['upper_constraint_vector'].append( + space.max_size) + + return constraint_equation + + def _merge_equations(self, eq1, eq2): + """Merges two equations into a single dictionary of equations. + + :param eq1: equation dictionary, where key is a name of equation and + value is a vector or matrix + :param eq2: same as eq1 + :return: merged equation + """ + result = {} + all_keys = set(eq1.keys() + eq2.keys()) + for key in all_keys: + if eq2.get(key) and eq1.get(key): + # Merge if both have values + result[key] = eq1[key] + eq2[key] + elif eq2.get(key): + result[key] = eq2[key] + elif eq1.get(key): + result[key] = eq1[key] + + return result + + def _make_disk_size_constraints(self): + """Creates equations based on disk sizes. + + So solver will not allocate more then "disk size" space for each disk. + + In case of 2 spaces and 3 disks the result should be: + [[1, 1, 0, 0, 0, 0], + [0, 0, 1, 1, 0, 0], + [0, 0, 0, 0, 1, 1]] + + Explanation of the first row + [1, - x1 multiplier, size of space 1 on the first disk + 1, - x2 multiplier, size of space 2 on the first disk + 0, - x3 multiplier, size of space 1 on 2nd disk, 0 for the first + 0, - x4 multiplier, size of space 2 on 2nd disk, 0 for the first + 0, - x5 multiplier, size of space 1 on 3rd disk, 0 for the first + 0] - x6 multiplier, size of space 2 on 3rd disk, 0 for the first + + :return: equations, where key is a name of equation, value is a list + or vector + """ + constraint_equation = { + 'upper_constraint_matrix': [], + 'upper_constraint_vector': []} + + for disk_idx in range(self.disks_len): + row = self._make_matrix_row() + + for space_idx, space in enumerate(self.spaces): + row[disk_idx * self.spaces_len + space_idx] = 1 + + constraint_equation['upper_constraint_matrix'].append(row) + constraint_equation['upper_constraint_vector'].append( + self.disks[disk_idx].size) + + return constraint_equation + + def _make_weight_constraints(self): + """Refresh weight. + + Create weight constraints for spaces which have same + max constraint or for those which don't have it at all. + + Lets say, second space is equal to the third, as the result + we will have next equation: + 0 * x1 + (1 / weight) * x2 + (-1 / weight) * x3 + + 0 * x4 + (1 / weight) * x5 + (-1 / weight) * x6 = 0 + + See "Weight" section in the documentation for details: + http://bareon-allocator.readthedocs.org/en + /latest/architecture.html#weight + + TODO(eli): it should be not equality, but inequality with some + range, so we will not get fails every time exact constraint cannot be + satisfied. + """ + weight_equations = { + 'equality_constraint_matrix': [], + 'equality_constraint_vector': []} + + weight_spaces_sets = self._get_spaces_sets_by( + self.weight_sets_criteria) + + for spaces_set in weight_spaces_sets: + # Don't set weight if there is less than one space in the set + if len(spaces_set) < 2: + continue + + first_weight = spaces_set[0].weight + first_space_idx = self.spaces.index(spaces_set[0]) + for space in spaces_set[1:]: + row = self._make_matrix_row() + + # If weight is 0, it doesn't make sense to set for such + # space a weight + if space.weight == 0: + continue + + space_idx = self.spaces.index(space) + + for disk_idx in range(self.disks_len): + row_i = disk_idx * len(self.spaces) + row[row_i + first_space_idx] = 1.0 / first_weight + row[row_i + space_idx] = -1.0 / space.weight + + weight_equations['equality_constraint_matrix'].append(row) + weight_equations['equality_constraint_vector'].append(0) + + return weight_equations + + def _make_objective_function_coefficient(self): + """Creates objective function coefficients. + + We want spaces to be allocated on disks in order which user + specified them in the schema. In order to do that, we set + coefficients higher for those spaces which defined earlier in the + list. + + :return: a vector of coefficients + """ + + # Instead of just Integer sequence special type of sequence is being + # used, see documentation [1] for details. + # Every order coefficient should be between 0 and 1 (not included), + # in order to aviod having 1st element equal to 1, sequence should be + # started from 2nd element. + # + # [1] http://bareon-allocator.readthedocs.org/en + # /latest/architecture.html#ordering + seq = CrossSumInequalitySequence(self.x_amount + 1) + next(seq, None) + coefficients = [1.0 / i for i in seq] + + space_sets = self._get_spaces_sets_by(['best_with_disks']) + no_best_disks = self._get_empty_sets_disks_ids(['best_with_disks']) + + for i_set, space_set in enumerate(space_sets): + for space in space_set: + s_i = self.spaces.index(space) + + for d_i, disk in enumerate(self.disks): + c_i = self.spaces_len * d_i + s_i + + # Set constant for none_order spaces + if space.none_order: + coefficients[c_i] = self.NONE_ORDER_COEFFICIENT + continue + + # If space does not belong to any set, order coefficient + # will be left without any additional coefficients. + if (space.best_with_disks and + disk.id in space.best_with_disks): + # If the space has "best disks" and current disk is + # in best disks list, add coefficient. + coefficients[c_i] += self.SET_COEFFICIENT + elif (not space.best_with_disks and + disk.id in no_best_disks): + # If the space does *not* have "best disks" and + # current disk is not in the list of "best disks" of + # any space, add set coefficient. + coefficients[c_i] += self.SET_COEFFICIENT + + # By default the algorithm tries to minimize the solution + # we should invert sign, in order to make it a maximization + # function, because we want disks to be maximally allocated. + return [-c for c in coefficients] + + def _get_empty_sets_disks_ids(self, criteria): + """Get disks indexes which do not belong to set of any spaces. + + :param criteria: a list of strings, with criteria by which sets has + to be created + :return: a list of disks indexes + """ + all_disks_ids = [d.id for d in self.disks] + used_disks_ids = [] + + for k, space in self._get_sets_by(criteria): + if k[0]: + used_disks_ids.extend(list(k[0])) + + return list(set(all_disks_ids) - set(used_disks_ids)) + + def _get_spaces_sets_by(self, criteria): + """Get all spaces which are used for sets. + + :param criteria: a list of strings with attributes by which sets has + to be created + :return: a list of spaces lists, where each list item is represents + a set + """ + return [i[1] for i in self._get_sets_by(criteria)] + + def _get_sets_by(self, criteria): + """Makes sets based on criteria from space attributes. + + :param criteria: a list of strings with attributes by which sets has + to be created + :return: a list of tuples, where first item are criteria, second + item is a list of spaces + """ + def get_values(space): + return [getattr(space, c, None) for c in criteria] + + grouped_spaces = itertools.groupby( + sorted(self.spaces, key=get_values), + key=get_values) + + return [(k, list(v)) for k, v in grouped_spaces] + + def _make_matrix_row(self): + """Make a matrix row + + :return: a vector where all the items are 0 + """ + return [0] * self.x_amount diff --git a/bareon_allocator/solvers/linear_programming_scipy_solver.py b/bareon_allocator/solvers/linear_programming_scipy_solver.py new file mode 100644 index 0000000..1255605 --- /dev/null +++ b/bareon_allocator/solvers/linear_programming_scipy_solver.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from scipy.optimize import linprog + +from bareon_allocator import errors +from bareon_allocator.solvers import BaseSolver +from bareon_allocator.solvers import utils + + +class LinearProgrammingScipySolver(BaseSolver): + """Linear programming allocator. + + Use Linear Programming method [0] (the method itself has nothing to do + with computer-programming) in order to formulate and solve the problem + of spaces allocation on disks, with the best outcome. + + In this implementation scipy is being used since it already implements + simplex algorithm to find the best feasible solution. + + [0] https://en.wikipedia.org/wiki/Linear_programming + [1] http://docs.scipy.org/doc/scipy-0.16.0/reference/generated + /scipy.optimize.linprog.html + [2] https://en.wikipedia.org/wiki/Simplex_algorithm + """ + + def solve(self): + """Solves linear program. + + :return: solution vector + """ + lp_solution = linprog( + self.linear_program.objective_function_coefficients, + A_eq=self.linear_program.equality_constraint_matrix or None, + b_eq=self.linear_program.equality_constraint_vector or None, + A_ub=self._make_upper_constraint_matrix() or None, + b_ub=self._make_upper_constraint_vector() or None, + bounds=self.linear_program.bounds, + options={"disp": False}) + + self._check_errors(lp_solution) + + # Naive implementation of getting integer result + # from a linear programming algorithm, MIP + # (mixed integer programming) should be considered + # instead, but it may have a lot of problems (solution + # of such equations is NP-hard in some cases), + # for our practical purposes it's enough to round + # the number down, in this case we may get `n` megabytes + # unallocated, where n is len(spaces) * len(disks) + solution_vector = utils.round_vector_down(lp_solution.x) + + return solution_vector + + def _check_errors(self, solution): + """Checks if solution is not found. + + :param solution: solution object from scipy + :raises: errors.NoSolutionFound if solution is not found + """ + if not solution.success: + raise errors.NoSolutionFound( + 'Allocation is not possible ' + 'with specified constraints: {0}'.format(solution.message)) + + def _make_upper_constraint_matrix(self): + """Merges lower constraint matrix into upper.""" + upper_constraint_matrix = [] + if self.linear_program.upper_constraint_matrix: + upper_constraint_matrix.extend( + self.linear_program.upper_constraint_matrix) + + if self.linear_program.lower_constraint_matrix: + # Swap sign for lower constraint matrix in order to make it + # upper bound instead of lower bound + upper_constraint_matrix.extend( + [-i for i in row] for row in + self.linear_program.lower_constraint_matrix) + + return upper_constraint_matrix + + def _make_upper_constraint_vector(self): + """Merges lower constraint vector into upper.""" + upper_constraint_vector = [] + if self.linear_program.upper_constraint_vector: + upper_constraint_vector.extend( + self.linear_program.upper_constraint_vector) + + if self.linear_program.lower_constraint_vector: + # Swap sign for items in the vector to make it upper bound + # instead of lower bound + upper_constraint_vector.extend( + [-i for i in self.linear_program.lower_constraint_vector]) + + return upper_constraint_vector diff --git a/bareon_allocator/solvers/utils.py b/bareon_allocator/solvers/utils.py new file mode 100644 index 0000000..a3148d3 --- /dev/null +++ b/bareon_allocator/solvers/utils.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import math + + +def round_vector_down(vector): + """Rounds items in the vector down. + + :param vector: vector of float numbers + :return: a list of integers + """ + return [int(math.floor(f)) for f in vector] diff --git a/bareon_allocator/tests/test_bareon_dynamic_allocator.py b/bareon_allocator/tests/test_bareon_dynamic_allocator.py deleted file mode 100644 index 0e8bbb6..0000000 --- a/bareon_allocator/tests/test_bareon_dynamic_allocator.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -test_bareon_allocator ----------------------------------- - -Tests for `bareon_allocator` module. -""" - -from bareon_allocator.tests import base - - -class TestBareon_dynamic_allocator(base.TestCase): - - def test_something(self): - pass diff --git a/bareon_allocator/tests/test_objects_disk.py b/bareon_allocator/tests/test_objects_disk.py new file mode 100644 index 0000000..1e57a17 --- /dev/null +++ b/bareon_allocator/tests/test_objects_disk.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from bareon_allocator.objects import Disk +from bareon_allocator.tests import base + + +class TestObjectsDisk(base.TestCase): + + def test_object_creation(self): + disk = Disk(id=10, size=42) + self.assertEqual(disk.id, 10) + self.assertEqual(disk.size, 42) diff --git a/bareon_allocator/tests/test_objects_space.py b/bareon_allocator/tests/test_objects_space.py new file mode 100644 index 0000000..1440d6d --- /dev/null +++ b/bareon_allocator/tests/test_objects_space.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from bareon_allocator import errors +from bareon_allocator.objects import Space +from bareon_allocator.tests import base + + +class TestObjectsSpace(base.TestCase): + + def test_object_creation(self): + space = Space(id=10, + min_size=1, + max_size=2, + type='lv', + best_with_disks=[1, 2, 3]) + self.assertEqual(space.id, 10) + self.assertEqual(space.min_size, 1) + self.assertEqual(space.max_size, 2) + self.assertEqual(space.type, 'lv') + self.assertEqual(space.best_with_disks, [1, 2, 3]) + self.assertEqual(space.weight, 1) + self.assertEqual(space.none_order, False) + + def test_size_sets_min_and_max(self): + space = Space(id=10, type='lv', size=15) + self.assertEqual(space.min_size, 15) + self.assertEqual(space.max_size, 15) + + def test_fail_if_no_type(self): + self.assertRaises(errors.InvalidData, Space, id=11) diff --git a/bareon_allocator/tests/test_parsers_dynamic_schema_parser.py b/bareon_allocator/tests/test_parsers_dynamic_schema_parser.py new file mode 100644 index 0000000..5ea7ecc --- /dev/null +++ b/bareon_allocator/tests/test_parsers_dynamic_schema_parser.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from bareon_allocator.parsers import DynamicSchemaParser +from bareon_allocator.tests import base + + +class TestParsersDynamicSchemaParser(base.TestCase): + + def setUp(self): + super(TestParsersDynamicSchemaParser, self).setUp() + hw_info = { + 'disks': [ + {'id': 'sda', 'size': 100}, + {'id': 'sdb', 'size': 42}, + {'id': 'sdc', 'size': 42}]} + schema = [ + {'id': 'lv1', + 'type': 'lv', + 'max_size': 1}, + {'id': 'lv2', + 'type': 'lv', + 'max_size': 1, + 'best_with_disks': 'yaql=$.disks.where($.size=42)'}, + {'id': 'vg1', + 'type': 'vg'}] + self.dynamic_schema_parser = DynamicSchemaParser(hw_info, schema) + + def test_unallocated_is_added(self): + unallocated = filter(lambda s: s.id == 'unallocated', + self.dynamic_schema_parser.spaces) + + self.assertEqual(len(unallocated), 1) + self.assertEqual(unallocated[0].type, 'unallocated') + self.assertEqual(unallocated[0].none_order, True) + self.assertEqual(unallocated[0].weight, 0) + + def test_aggregation_spaces_are_not_in_the_list(self): + spaces = filter(lambda d: d.type == 'vg', + self.dynamic_schema_parser.spaces) + self.assertEqual(len(spaces), 0) + + def test_sets_best_with_disks_ids(self): + spaces = filter(lambda s: s.id == 'lv2', + self.dynamic_schema_parser.spaces) + + self.assertEqual(len(spaces), 1) + self.assertEqual(spaces[0].best_with_disks, {'sdb', 'sdc'}) diff --git a/bareon_allocator/tests/test_parsers_expressions.py b/bareon_allocator/tests/test_parsers_expressions.py new file mode 100644 index 0000000..37e94c3 --- /dev/null +++ b/bareon_allocator/tests/test_parsers_expressions.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from bareon_allocator.parsers import ExpressionsParser +from bareon_allocator.tests import base + + +class TestParsersExpressions(base.TestCase): + + def test_substitutes_value_recursively(self): + parsed = ExpressionsParser( + [{'key1': 'key2'}, + {'list': [{'list_key': 'yaql=$.some_key'}]}], + {'some_key': 'some_value'}).parse() + + self.assertEqual( + parsed, + [{'key1': 'key2'}, + {'list': [{'list_key': 'some_value'}]}]) diff --git a/bareon_allocator/tests/test_solvers_linear_program.py b/bareon_allocator/tests/test_solvers_linear_program.py new file mode 100644 index 0000000..127fe6e --- /dev/null +++ b/bareon_allocator/tests/test_solvers_linear_program.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from bareon_allocator.solvers import LinearProgram +from bareon_allocator.tests import base + + +class TestSolversLinearProgram(base.TestCase): + + def setUp(self): + super(TestSolversLinearProgram, self).setUp() + self.lp = LinearProgram( + x_amount=3, + objective_function_coefficients=[1, 0, 0], + + equality_constraint_matrix=[[1, 2, 3], [4, 5, 6]], + lower_constraint_matrix=[[7, 8, 9], [10, 11, 12]], + upper_constraint_matrix=[[13, 14, 15], [16, 17, 18]], + + equality_constraint_vector=[1, 2, 3], + lower_constraint_vector=[3, 4, 5], + upper_constraint_vector=[6, 7, 8]) + + def test_values_are_set(self): + self.assertEqual(self.lp.x_amount, 3) + self.assertEqual( + self.lp.objective_function_coefficients, + [1, 0, 0]) + + self.assertEqual( + self.lp.equality_constraint_matrix, + [[1, 2, 3], [4, 5, 6]]) + self.assertEqual( + self.lp.lower_constraint_matrix, + [[7, 8, 9], [10, 11, 12]]) + self.assertEqual( + self.lp.upper_constraint_matrix, + [[13, 14, 15], [16, 17, 18]]) + + self.assertEqual( + self.lp.equality_constraint_vector, + [1, 2, 3]) + self.assertEqual( + self.lp.lower_constraint_vector, + [3, 4, 5]) + self.assertEqual( + self.lp.upper_constraint_vector, + [6, 7, 8]) + + def test_default_values_are_set(self): + self.assertEqual(self.lp.lp_type, self.lp.LP_TYPE_LP) + self.assertEqual(self.lp.objective_function_optimization_type, + self.lp.MAXIMIZE) + self.assertEqual(self.lp.bounds, [(0, None), (0, None), (0, None)]) diff --git a/bareon_allocator/tests/test_solvers_linear_program_creator.py b/bareon_allocator/tests/test_solvers_linear_program_creator.py new file mode 100644 index 0000000..f3f2b67 --- /dev/null +++ b/bareon_allocator/tests/test_solvers_linear_program_creator.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from bareon_allocator.solvers import LinearProgramCreator +from bareon_allocator.tests import base + + +class TestSolversLinearProgramCreator(base.TestCase): + + def create_lp(self, spaces_info=[], disks_info=[]): + dynamic_schema_mock = mock.MagicMock(spaces=[], disks=[]) + for s in spaces_info: + dynamic_schema_mock.spaces.append(mock.MagicMock(**s)) + + for d in disks_info: + dynamic_schema_mock.disks.append(mock.MagicMock(**d)) + + return LinearProgramCreator(dynamic_schema_mock).linear_program() + + def assert_lower_eq_exists(self, lp, eq, value): + self.assert_eq_and_value( + lp.lower_constraint_matrix, + lp.lower_constraint_vector, + eq, + value) + + def assert_upper_eq_exists(self, lp, eq, value): + self.assert_eq_and_value( + lp.upper_constraint_matrix, + lp.upper_constraint_vector, + eq, + value) + + def assert_eq_exists(self, lp, eq, value): + self.assert_eq_and_value( + lp.equality_constraint_matrix, + lp.equality_constraint_vector, + eq, + value) + + def assert_eq_and_value(self, eq_list, eq_vector, expected_eq, value): + eq = None + i_eq = None + for i, _eq in enumerate(eq_list): + if _eq == expected_eq: + eq = _eq + i_eq = i + break + + self.assertIsNotNone(eq, 'Cannot find equation') + self.assertEqual(eq_vector[i_eq], value, + 'Value of equation does not match to expected') + + def test_min_size_equations(self): + lp = self.create_lp( + spaces_info=[ + {'min_size': 10}, + {'min_size': 20}, + {'min_size': 0}], + disks_info=[ + {'id': 'sda', 'size': 50}, + {'id': 'sda', 'size': 60}]) + self.assert_lower_eq_exists(lp, [1, 0, 0, 1, 0, 0], 10) + self.assert_lower_eq_exists(lp, [0, 1, 0, 0, 1, 0], 20) + self.assert_lower_eq_exists(lp, [0, 0, 1, 0, 0, 1], 0) + + def test_max_size_equations(self): + lp = self.create_lp( + spaces_info=[ + {'max_size': 10}, + {'max_size': 20}, + {'min_size': 0}], + disks_info=[ + {'id': 'sda', 'size': 50}, + {'id': 'sda', 'size': 60}]) + self.assert_upper_eq_exists(lp, [1, 0, 0, 1, 0, 0], 10) + self.assert_upper_eq_exists(lp, [0, 1, 0, 0, 1, 0], 20) + + def test_disk_size_equations(self): + lp = self.create_lp( + spaces_info=[ + {'max_size': 10}, + {'max_size': 20}, + {'min_size': 0}], + disks_info=[ + {'id': 'sda', 'size': 50}, + {'id': 'sda', 'size': 60}]) + self.assert_upper_eq_exists(lp, [1, 1, 1, 0, 0, 0], 50) + self.assert_upper_eq_exists(lp, [0, 0, 0, 1, 1, 1], 60) + + def test_weight_eq(self): + lp = self.create_lp( + spaces_info=[ + {'id': 'v1', 'min_size': 20, 'max_size': None, + 'best_with_disks': [], 'weight': 10}, + {'id': 'v2', 'min_size': 20, 'max_size': None, + 'best_with_disks': [], 'weight': 5}, + {'id': 'v3', 'min_size': 30, 'max_size': None, + 'best_with_disks': ['sda'], 'weight': 1}, + {'id': 'v4', 'min_size': 30, 'max_size': None, + 'best_with_disks': ['sda'], 'weight': 1}], + disks_info=[ + {'id': 'sda', 'size': 100}, + {'id': 'sdb', 'size': 200}, + {'id': 'sdc', 'size': 300}]) + + self.assert_eq_exists( + lp, + [0.1, -0.2, 0, 0, + 0.1, -0.2, 0, 0, + 0.1, -0.2, 0, 0], + 0) + + self.assert_eq_exists( + lp, + [0, 0, 1.0, -1.0, + 0, 0, 1.0, -1.0, + 0, 0, 1.0, -1.0], + 0) + + def test_objective_function_equation(self): + lp = self.create_lp( + spaces_info=[ + {'id': 'v0', 'min_size': 20, 'max_size': None, + 'best_with_disks': [], + 'weight': 10, 'none_order': False}, + {'id': 'v1', 'min_size': 20, 'max_size': None, + 'best_with_disks': [], + 'weight': 5, 'none_order': False}, + {'id': 'v2', 'min_size': 30, 'max_size': None, + 'best_with_disks': ['sda'], + 'weight': 1, 'none_order': False}, + {'id': 'v3', 'min_size': 30, 'max_size': None, + 'best_with_disks': ['sda'], + 'weight': 1, 'none_order': False}], + disks_info=[ + {'id': 'sda', 'size': 100}, + {'id': 'sdb', 'size': 200}, + {'id': 'sdc', 'size': 300}]) + + seq = [2, 4, 6, 9, 12, 16, 20, 25, 30, 36, 42, 49] + reverse_seq = [-1.0 / s for s in seq] + + weight_indexes = [ + 2, # v2, sda + 3, # v3, sda + 4, # v0, sdb + 5, # v1, sdb + 8, # v0, sdc + 9] # v1, sdc + + for idx in weight_indexes: + # Substitute "set" coefficient + reverse_seq[idx] -= 2 + + self.assertEqual( + lp.objective_function_coefficients, + reverse_seq) diff --git a/bareon_allocator/tests/test_solvers_linear_programming_scipy_solver.py b/bareon_allocator/tests/test_solvers_linear_programming_scipy_solver.py new file mode 100644 index 0000000..4a3861e --- /dev/null +++ b/bareon_allocator/tests/test_solvers_linear_programming_scipy_solver.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from bareon_allocator import errors +from bareon_allocator.solvers.linear_program import LinearProgram +from bareon_allocator.solvers.linear_programming_scipy_solver \ + import LinearProgrammingScipySolver +from bareon_allocator.tests import base + + +class TestSolversLinearProgrammingScipySolver(base.TestCase): + + def test_solves_lp(self): + # x = 1 + # y = 1 + # z + j >= 2 + # x + y + z + j <= 4 + lp = LinearProgram( + equality_constraint_matrix=[[1, 0, 0, 0], [0, 1, 0, 0]], + equality_constraint_vector=[1, 1], + lower_constraint_matrix=[[0, 0, 1, 1]], + lower_constraint_vector=[2], + upper_constraint_matrix=[[1, 1, 1, 1]], + upper_constraint_vector=[4], + objective_function_coefficients=[0, 0, 0, 0], + x_amount=4) + + solver = LinearProgrammingScipySolver(lp) + self.assertEqual( + solver.solve(), + [1, 1, 2, 0]) + + def test_raises_error(self): + # 0 + 0 = 2 + lp = LinearProgram( + equality_constraint_matrix=[[0, 0]], + equality_constraint_vector=[1], + objective_function_coefficients=[0, 0], + x_amount=2) + + solver = LinearProgrammingScipySolver(lp) + self.assertRaises(errors.NoSolutionFound, solver.solve) diff --git a/bareon_allocator/tests/test_solvers_utils.py b/bareon_allocator/tests/test_solvers_utils.py new file mode 100644 index 0000000..798f265 --- /dev/null +++ b/bareon_allocator/tests/test_solvers_utils.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from bareon_allocator.solvers.utils import round_vector_down +from bareon_allocator.tests import base + + +class TestSolversUtils(base.TestCase): + + def test_round_down(self): + self.assertEqual( + round_vector_down([1.9, 2.0, 3.1]), + [1, 2, 3]) diff --git a/doc/source/architecture.rst b/doc/source/architecture.rst index 2c3a31d..5d9fc1f 100644 --- a/doc/source/architecture.rst +++ b/doc/source/architecture.rst @@ -673,7 +673,7 @@ or #. Build sets according to selected disks, in our case we have two sets, **hdd** and **ssd** disks. #. For spaces which belong to specific set of disks add **1** to a coefficient which represents this space on a disk from the set. -#. If space does not belong to the set of disks, add **0**. +#. Spaces which do not belong to any disks sets are assigned to set of disks which is left, in our case it is **hdd** disks set. To make sure that spaces are always (unless size constraints are not violated) allocated on the disks which they best suited with, we automatically add a special artificial volume **unallocated**, whose coefficient is always **1**, and in this case we should change