diff --git a/bareon_dynamic_allocator/allocators.py b/bareon_dynamic_allocator/allocators.py index 23ebb91..5eb98d9 100644 --- a/bareon_dynamic_allocator/allocators.py +++ b/bareon_dynamic_allocator/allocators.py @@ -1,4 +1,6 @@ -# Copyright 2015 Mirantis, Inc. +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -8,19 +10,19 @@ # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import itertools import math -import six import numpy as np -from termcolor import colored +import six + from oslo_log import log from scipy.optimize import linprog -from scipy.ndimage.interpolation import shift +from termcolor import colored from bareon_dynamic_allocator import errors from bareon_dynamic_allocator.parser import Parser @@ -38,7 +40,8 @@ def shift(arr, steps, val=0): def grouper(iterable, n, fillvalue=None): - """Collect data into fixed-length chunks or blocks + """Collect data into fixed-length chunks or blocks. + Source: https://docs.python.org/2/library/itertools.html#recipes """ args = [iter(iterable)] * n @@ -46,11 +49,10 @@ def grouper(iterable, n, fillvalue=None): def format_x_vector(coefficients, num=0): - return '\n' + '\n'.join( + return '\n{0}\n'.format('\n'.join( [' + '.join(group) - for group in grouper( - ['({0:+.5f} * x{1})'.format(c, i) - for i, c in enumerate(coefficients)], num)]) + '\n' + for group in grouper(['({0:+.5f} * x{1})'.format(c, i) + for i, c in enumerate(coefficients)], num)])) def format_equation(matrix, vector, row_len): @@ -77,7 +79,6 @@ def format_equation(matrix, vector, row_len): return '\n'.join(equation) - class Disk(object): def __init__(self, **kwargs): @@ -119,7 +120,8 @@ class DynamicAllocator(object): Parser(schema, hw_info).parse(), hw_info) LOG.debug('Rendered spaces schema: \n%s', rendered_spaces) - self.spaces = [Space(**space) for space in rendered_spaces if space['type'] != 'vg'] + self.spaces = [Space(**space) + for space in rendered_spaces if space['type'] != 'vg'] # Unallocated is required in order to be able to specify # spaces with only minimal @@ -140,7 +142,9 @@ class DynamicAllocator(object): return sizes def convert_disks_to_indexes(self, spaces, hw_info): - """Convert disks which are specified in `best_with_disks` + """Convert disks to indexes. + + Convert disks which are specified in `best_with_disks` to a list of indexes in `disks` list. """ for i, space in enumerate(spaces): @@ -159,7 +163,9 @@ class DynamicAllocator(object): class DynamicAllocationLinearProgram(object): - """Use Linear Programming method [0] (the method itself has nothing to do + """Linear programming allocator. + + Use Linear Programming method [0] (the method itself has nothing to do with computer-programming) in order to formulate and solve the problem of spaces allocation on disks, with the best outcome. @@ -214,7 +220,7 @@ class DynamicAllocationLinearProgram(object): # disk we have len(spaces) * len(disks) sizes self.x_amount = len(self.disks) * len(self.spaces) - # TODO: has to be refactored + # TODO(eli): has to be refactored # Here we store indexes for bounds and equation # matrix, in order to be able to change it on # refresh @@ -231,7 +237,8 @@ class DynamicAllocationLinearProgram(object): upper_bound_vector = self._make_upper_bound_constraint_vector() or None LOG.debug('Objective function coefficients human-readable:\n%s\n', - format_x_vector(self.objective_function_coefficients, len(self.spaces))) + format_x_vector(self.objective_function_coefficients, + len(self.spaces))) LOG.debug('Equality equation:\n%s\n', format_equation( @@ -245,7 +252,8 @@ class DynamicAllocationLinearProgram(object): len(self.spaces))) for weight_for_sets in self.weight_set_mapping: - LOG.debug('Parameters for spaces set formation: %s', weight_for_sets) + LOG.debug('Parameters for spaces set formation: %s', + weight_for_sets) self._set_spaces_sets_by(weight_for_sets) solution = linprog( self.objective_function_coefficients, @@ -321,8 +329,8 @@ class DynamicAllocationLinearProgram(object): return [getattr(space, c, None) for c in criteria] grouped_spaces = itertools.groupby( - sorted(self.spaces, key=get_values), - key=get_values) + sorted(self.spaces, key=get_values), + key=get_values) return [(k, list(v)) for k, v in grouped_spaces] @@ -330,7 +338,9 @@ class DynamicAllocationLinearProgram(object): self.weight_spaces_sets = self._get_spaces_sets_by(criteria) def _refresh_weight(self): - """Create weight constraints for spaces which have same + """Refresh weight. + + Create weight constraints for spaces which have same max constraint or for those which don't have it at all. Lets say, second's space is equal to max of the third and fourth, @@ -356,34 +366,44 @@ class DynamicAllocationLinearProgram(object): row = self._make_matrix_row() weight = getattr(space, 'weight', DEFAULT_WEIGHT) - # If weight is 0, it doesn't make sense to set for such space a weight + # If weight is 0, it doesn't make sense to set for such + # space a weight if weight == 0: continue space_idx = self.spaces.index(space) for disk_idx in range(len(self.disks)): - row[disk_idx * len(self.spaces) + first_space_idx] = 1 / first_weight - row[disk_idx * len(self.spaces) + space_idx] = -1 / weight + row_i = disk_idx * len(self.spaces) + row[row_i + first_space_idx] = 1 / first_weight + row[row_i + space_idx] = -1 / weight - self.weight_equation_indexes.append(len(self.equality_constraint_matrix) - 1) + self.weight_equation_indexes.append( + len(self.equality_constraint_matrix) - 1) self.equality_constraint_matrix.append(row) - self.equality_constraint_vector = np.append(self.equality_constraint_vector, 0) + self.equality_constraint_vector = np.append( + self.equality_constraint_vector, + 0) def _make_matrix_row(self): return np.zeros(self.x_amount) def _make_upper_bound_constraint_matrix(self): - """Upper bound constraint matrix consist of upper bound - matrix and lower bound matrix witch changed sign + """Creates upper bound constraint matrix. + + Upper bound constraint matrix consist of upper bound + matrix and lower bound matrix witch changed sign. """ return (self.upper_bound_constraint_matrix + - [[-i for i in row] for row in self.lower_bound_constraint_matrix]) + [[-i for i in row] + for row in self.lower_bound_constraint_matrix]) def _make_upper_bound_constraint_vector(self): - """Upper bound constraint vector consist of upper bound - and lower bound, with changed sign + """Create upper bound constraint vector. + + Upper bound constraint vector consist of upper bound and + lower bound, with changed sign. """ return (self.upper_bound_constraint_vector + [-i for i in self.lower_bound_constraint_vector]) @@ -391,10 +411,14 @@ class DynamicAllocationLinearProgram(object): def _convert_solution(self, solution_vector): result = [] - spaces_grouped_by_disk = list(grouper(solution_vector, len(self.spaces))) + spaces_grouped_by_disk = list(grouper( + solution_vector, + len(self.spaces))) for disk_i in range(len(self.disks)): disk_id = self.disks[disk_i].id - disk = {'disk_id': disk_id, 'size': self.disks[disk_i].size, 'spaces': []} + disk = {'disk_id': disk_id, + 'size': self.disks[disk_i].size, + 'spaces': []} spaces_for_disk = spaces_grouped_by_disk[disk_i] for space_i, space_size in enumerate(spaces_for_disk): @@ -410,7 +434,9 @@ class DynamicAllocationLinearProgram(object): for d in disks: # Initialize constraints, each row in the matrix should # be equal to size of the disk - self.equality_constraint_vector = np.append(self.equality_constraint_vector, d.size) + self.equality_constraint_vector = np.append( + self.equality_constraint_vector, + d.size) # Initialize the matrix # In case of 2 spaces and 3 disks the result should be: @@ -432,7 +458,10 @@ class DynamicAllocationLinearProgram(object): for _ in range(len(disks)): self.equality_constraint_matrix.append(equality_matrix_row) - equality_matrix_row = shift(equality_matrix_row, len(spaces), val=0) + equality_matrix_row = shift( + equality_matrix_row, + len(spaces), + val=0) # Size of each space should be more or equal to 0 for _ in range(self.x_amount): @@ -448,9 +477,9 @@ class DynamicAllocationLinearProgram(object): # higher for those spaces which defined earlier # in the list - # TODO describe why we should use special sequence + # TODO(eli): describe why we should use special sequence # as order coefficients - coefficients = [1.0/i for i in CrossSumInequalitySequence(c_amount)] + coefficients = [1.0 / i for i in CrossSumInequalitySequence(c_amount)] NONE_ORDER_COEFF = 1 SET_COEFF = 2 @@ -484,7 +513,8 @@ class DynamicAllocationLinearProgram(object): coefficients[c_i] += SET_COEFF else: # If current disk is not in the set, set it to 0 - # TODO isn't it better to leave there order coefficient? + # TODO(eli): isn't it better to leave there order + # coefficient? # coefficients[c_i] = 0 pass else: diff --git a/bareon_dynamic_allocator/cmd.py b/bareon_dynamic_allocator/cmd.py index 02456cd..a9a833e 100644 --- a/bareon_dynamic_allocator/cmd.py +++ b/bareon_dynamic_allocator/cmd.py @@ -8,7 +8,7 @@ # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. @@ -17,8 +17,8 @@ import sys from oslo_config import cfg from oslo_log import log -from bareon_dynamic_allocator import utils from bareon_dynamic_allocator.allocators import DynamicAllocator +from bareon_dynamic_allocator import utils from bareon_dynamic_allocator import viewer @@ -84,12 +84,12 @@ def save_result(data, output_file): def validate_schema(schema): - # TODO should be implemented + # TODO(eli): should be implemented return schema def validate_hw_info(hw_info): - # TODO should be implemented + # TODO(eli): should be implemented return hw_info diff --git a/bareon_dynamic_allocator/errors.py b/bareon_dynamic_allocator/errors.py index 818862d..600e938 100644 --- a/bareon_dynamic_allocator/errors.py +++ b/bareon_dynamic_allocator/errors.py @@ -8,10 +8,11 @@ # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + class BareonDynamicAllocator(Exception): pass diff --git a/bareon_dynamic_allocator/parser.py b/bareon_dynamic_allocator/parser.py index ccdb9e2..40aaafd 100644 --- a/bareon_dynamic_allocator/parser.py +++ b/bareon_dynamic_allocator/parser.py @@ -8,15 +8,14 @@ # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import yaml -import yaql -import six import re +import six +import yaql def seq_iter(obj): @@ -27,7 +26,7 @@ def seq_iter(obj): for i in xrange(len(obj)): yield i, obj[i] - + class YAQLParser(object): engine_options = { 'yaql.limitIterators': 100, diff --git a/bareon_dynamic_allocator/sequences.py b/bareon_dynamic_allocator/sequences.py index 663fd57..9c2b526 100644 --- a/bareon_dynamic_allocator/sequences.py +++ b/bareon_dynamic_allocator/sequences.py @@ -8,7 +8,7 @@ # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. @@ -30,7 +30,7 @@ class BaseSequence(object): self.previous = self.current self.current += 1 return self.current - + class CrossSumInequalitySequence(BaseSequence): """An implementaion of a sequence from 1 to n @@ -52,7 +52,9 @@ class CrossSumInequalitySequence(BaseSequence): class FibonacciSequence(BaseSequence): - """Iterator over a sequence of Fibonacci numbers with n elements from 1 to n + """Generates fibinacci sequence + + Iterator over a sequence of Fibonacci numbers with n elements from 1 to n. """ def __init__(self, n): super(FibonacciSequence, self).__init__(n) @@ -64,5 +66,6 @@ class FibonacciSequence(BaseSequence): raise StopIteration else: self.n_current += 1 - self.previous, self.current = self.current, self.current + self.previous + self.previous, self.current = \ + self.current, self.current + self.previous return self.current diff --git a/bareon_dynamic_allocator/utils.py b/bareon_dynamic_allocator/utils.py index a2d91f2..cffe87e 100644 --- a/bareon_dynamic_allocator/utils.py +++ b/bareon_dynamic_allocator/utils.py @@ -8,7 +8,7 @@ # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. @@ -16,7 +16,8 @@ import yaml def parse_yaml(path): - """Parses yaml file + """Parses yaml file. + :param str path: path to the file :returns: dict or list """ diff --git a/bareon_dynamic_allocator/viewer.py b/bareon_dynamic_allocator/viewer.py index cbaf864..935e2c3 100644 --- a/bareon_dynamic_allocator/viewer.py +++ b/bareon_dynamic_allocator/viewer.py @@ -8,13 +8,13 @@ # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from pprint import pprint import svgwrite -from svgwrite import cm, mm +from svgwrite import cm class StdoutViewer(object): @@ -36,7 +36,9 @@ class SVGViewer(object): SPACES_X_INTERVAL = 2 STYLE = "fill:{color};stroke:black;stroke-width:5;" - def __init__(self, disks_spaces_mapping, file_path='/tmp/bareon.svg', fit=False): + def __init__(self, disks_spaces_mapping, + file_path='/tmp/bareon.svg', fit=False): + self.disks_spaces_mapping = disks_spaces_mapping max_disk_size = max([i['size'] for i in disks_spaces_mapping]) self.width_multiplier = 450.0 / max_disk_size @@ -47,7 +49,8 @@ class SVGViewer(object): if fit: options = { 'preserveAspectRatio': 'none', - 'viewBox': "0 0 {0} {1}".format(svg_width, svg_height) if fit else '0 0 maxY maxX'} + 'viewBox': "0 0 {0} {1}".format(svg_width, svg_height) + if fit else '0 0 maxY maxX'} self.dwg = svgwrite.Drawing( filename=file_path, @@ -59,16 +62,26 @@ class SVGViewer(object): self.dwg.save() def _add_disk_with_spaces(self): - disk_g = self.dwg.add(self.dwg.g(id='disks-group', transform="translate({0}, {1})".format(30, 30))) + disk_g = self.dwg.add(self.dwg.g( + id='disks-group', + transform="translate({0}, {1})".format(30, 30))) for disk_idx, disk_w_spaces in enumerate(self.disks_spaces_mapping): disk_id = disk_w_spaces['disk_id'] size = disk_w_spaces['size'] - disk = disk_g.add(self.dwg.g(id=disk_id, transform="translate(0, {0})".format(disk_idx * self.DISKS_INTERVAL))) - disk.add(self.dwg.text(text='{0} size={1}'.format(disk_id, size), fill="black")) + disk = disk_g.add(self.dwg.g( + id=disk_id, + transform="translate(0, {0})".format( + disk_idx * self.DISKS_INTERVAL))) - disk_rect = disk.add(self.dwg.g(transform="translate({0}, {1})".format(0, 10), id='in-{0}'.format(disk_id))) + disk.add(self.dwg.text( + text='{0} size={1}'.format(disk_id, size), + fill="black")) + + disk_rect = disk.add(self.dwg.g( + transform="translate({0}, {1})".format(0, 10), + id='in-{0}'.format(disk_id))) disk_rect.add(self.dwg.rect( style=self.STYLE.format(color='#f5f5f5'), ry=5, @@ -85,16 +98,22 @@ class SVGViewer(object): rx=5, id=space['space_id'], insert=last_insert, - size=(self.width_multiplier * space['size'], self.SPACE_HEIGHT))) + size=(self.width_multiplier * space['size'], + self.SPACE_HEIGHT))) last_insert[0] += self.width_multiplier * space['size'] - spaces_lines = ['{0} size={1}'.format(space['space_id'], space['size']) for space in disk_w_spaces['spaces']] + spaces_lines = ['{0} size={1}'.format(space['space_id'], + space['size']) + for space in disk_w_spaces['spaces']] last_insert[0] = self.width_multiplier * disk_w_spaces['size'] last_insert[0] += 10 last_insert[1] += 20 for space_idx, space_line in enumerate(spaces_lines): palette = self.PALETTE[space_idx % len(self.PALETTE)] - disk_rect.add(self.dwg.text(insert=last_insert, text=space_line, fill=palette)) + disk_rect.add(self.dwg.text( + insert=last_insert, + text=space_line, + fill=palette)) last_insert[1] += 20 diff --git a/doc_generate_static.py b/doc_generate_static.py index 6165ca2..e3bf16d 100644 --- a/doc_generate_static.py +++ b/doc_generate_static.py @@ -8,47 +8,65 @@ # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import six import os +import six from glob import glob +from bareon_dynamic_allocator.allocators import DynamicAllocator from bareon_dynamic_allocator import utils from bareon_dynamic_allocator import viewer -from bareon_dynamic_allocator.allocators import DynamicAllocator -doc_schemas_path = os.path.join(os.path.dirname(os.path.realpath(__file__)) , 'doc', 'source', 'schemas') -doc_schemas_rst_path = os.path.join(os.path.dirname(os.path.realpath(__file__)) , 'doc', 'source', 'examples.rst') +doc_schemas_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'doc', + 'source', + 'schemas') +doc_schemas_rst_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'doc', + 'source', + 'examples.rst') def generate_svg_files(): result = {} - for dynamic_schema_path in sorted(glob(os.path.join(doc_schemas_path, '*_ds.yaml'))): - print 'Read file {0}'.format(dynamic_schema_path) + for dynamic_schema_path in sorted( + glob(os.path.join(doc_schemas_path, '*_ds.yaml'))): + print('Read file {0}'.format(dynamic_schema_path)) dynamic_schema = utils.parse_yaml(dynamic_schema_path) dynamic_schema_file_name = os.path.basename(dynamic_schema_path) dynamic_schema_name = os.path.splitext(dynamic_schema_file_name)[0] - for hw_info_path in sorted(glob(os.path.join(doc_schemas_path, '*_disk.yaml'))): - print 'Read file {0}'.format(hw_info_path) + for hw_info_path in sorted( + glob(os.path.join(doc_schemas_path, '*_disk.yaml'))): + print('Read file {0}'.format(hw_info_path)) hw_info_file_name = os.path.basename(hw_info_path) hw_info_name = os.path.splitext(hw_info_file_name)[0] hw_info = utils.parse_yaml(hw_info_path) - static_schema = DynamicAllocator(hw_info, dynamic_schema).generate_static() + static_schema = DynamicAllocator( + hw_info, + dynamic_schema).generate_static() - static_schema_name = '{0}_{1}.svg'.format(dynamic_schema_name, hw_info_name) + static_schema_name = '{0}_{1}.svg'.format( + dynamic_schema_name, + hw_info_name) result[static_schema_name[:-4]] = { - 'dynamic_schema': os.path.join('schemas', dynamic_schema_file_name), + 'dynamic_schema': os.path.join('schemas', + dynamic_schema_file_name), 'hw_info': os.path.join('schemas', hw_info_file_name), 'image': os.path.join('schemas', static_schema_name)} - viewer.SVGViewer(static_schema, file_path=os.path.join(doc_schemas_path, static_schema_name), fit=True).show_me() + viewer.SVGViewer(static_schema, + file_path=os.path.join(doc_schemas_path, + static_schema_name), + fit=True).show_me() rst_doc = """ =================== diff --git a/lab_mip.py b/lab_mip.py index 787d11a..1647e0d 100644 --- a/lab_mip.py +++ b/lab_mip.py @@ -12,8 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. - -from pulp import * +from pulp import * # flake8: noqa import sys x = [] @@ -52,7 +51,7 @@ prob += x1 + x2 <= 100, 'First disk' prob += x3 + x4 <= 100, 'Second disk' prob += x5 + x6 <= 100, 'Third disk' -prob += y1 + y3 + y5 == 2, 'Replication factor' +prob += y1 + y3 + y5 == 1, 'Replication factor' prob += x2 + x4 + x6 >= 10, 'Second min size' @@ -102,6 +101,7 @@ for i, x_ in enumerate(x): # solve the problem status = prob.solve(GLPK(msg=1)) + def print_vector(vector, prefix, n=2): for i, v in enumerate(vector): @@ -111,10 +111,11 @@ def print_vector(vector, prefix, n=2): else: sys.stdout.write('\n') -print + +print() print_vector(x, 'x') -print +print() print_vector(y, 'y') -print +print() print_vector(z, 'z', n=3) -print +print()