x
/
bareon-allocator
Code Issues Proposed changes

Fix pep8 errors

This commit is contained in:
Evgeniy L 2016-02-09 19:09:43 +03:00
parent e78a76ffdb
commit a99384200b
9 changed files with 156 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
bareon_dynamic_allocator/allocators.py
View File

@ -1,4 +1,6 @@
# Copyright 2015 Mirantis, Inc.
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -8,19 +10,19 @@
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import math
import six
import numpy as np
from termcolor import colored
import six
from oslo_log import log
from scipy.optimize import linprog
from scipy.ndimage.interpolation import shift
from termcolor import colored
from bareon_dynamic_allocator import errors
from bareon_dynamic_allocator.parser import Parser
@ -38,7 +40,8 @@ def shift(arr, steps, val=0):
def grouper(iterable, n, fillvalue=None):
"""Collect data into fixed-length chunks or blocks
"""Collect data into fixed-length chunks or blocks.
Source: https://docs.python.org/2/library/itertools.html#recipes
"""
args = [iter(iterable)] * n
@ -46,11 +49,10 @@ def grouper(iterable, n, fillvalue=None):
def format_x_vector(coefficients, num=0):
return '\n' + '\n'.join(
return '\n{0}\n'.format('\n'.join(
[' + '.join(group)
for group in grouper(
['({0:+.5f} * x{1})'.format(c, i)
for i, c in enumerate(coefficients)], num)]) + '\n'
for group in grouper(['({0:+.5f} * x{1})'.format(c, i)
for i, c in enumerate(coefficients)], num)]))
def format_equation(matrix, vector, row_len):
@ -77,7 +79,6 @@ def format_equation(matrix, vector, row_len):
return '\n'.join(equation)
class Disk(object):
def __init__(self, **kwargs):
@ -119,7 +120,8 @@ class DynamicAllocator(object):
Parser(schema, hw_info).parse(),
hw_info)
LOG.debug('Rendered spaces schema: \n%s', rendered_spaces)
self.spaces = [Space(**space) for space in rendered_spaces if space['type'] != 'vg']
self.spaces = [Space(**space)
for space in rendered_spaces if space['type'] != 'vg']
# Unallocated is required in order to be able to specify
# spaces with only minimal
@ -140,7 +142,9 @@ class DynamicAllocator(object):
return sizes
def convert_disks_to_indexes(self, spaces, hw_info):
"""Convert disks which are specified in `best_with_disks`
"""Convert disks to indexes.
Convert disks which are specified in `best_with_disks`
to a list of indexes in `disks` list.
"""
for i, space in enumerate(spaces):
@ -159,7 +163,9 @@ class DynamicAllocator(object):
class DynamicAllocationLinearProgram(object):
"""Use Linear Programming method [0] (the method itself has nothing to do
"""Linear programming allocator.
Use Linear Programming method [0] (the method itself has nothing to do
with computer-programming) in order to formulate and solve the problem
of spaces allocation on disks, with the best outcome.
@ -214,7 +220,7 @@ class DynamicAllocationLinearProgram(object):
# disk we have len(spaces) * len(disks) sizes
self.x_amount = len(self.disks) * len(self.spaces)
# TODO: has to be refactored
# TODO(eli): has to be refactored
# Here we store indexes for bounds and equation
# matrix, in order to be able to change it on
# refresh
@ -231,7 +237,8 @@ class DynamicAllocationLinearProgram(object):
upper_bound_vector = self._make_upper_bound_constraint_vector() or None
LOG.debug('Objective function coefficients human-readable:\n%s\n',
format_x_vector(self.objective_function_coefficients, len(self.spaces)))
format_x_vector(self.objective_function_coefficients,
len(self.spaces)))
LOG.debug('Equality equation:\n%s\n',
format_equation(
@ -245,7 +252,8 @@ class DynamicAllocationLinearProgram(object):
len(self.spaces)))
for weight_for_sets in self.weight_set_mapping:
LOG.debug('Parameters for spaces set formation: %s', weight_for_sets)
LOG.debug('Parameters for spaces set formation: %s',
weight_for_sets)
self._set_spaces_sets_by(weight_for_sets)
solution = linprog(
self.objective_function_coefficients,
@ -321,8 +329,8 @@ class DynamicAllocationLinearProgram(object):
return [getattr(space, c, None) for c in criteria]
grouped_spaces = itertools.groupby(
sorted(self.spaces, key=get_values),
key=get_values)
sorted(self.spaces, key=get_values),
key=get_values)
return [(k, list(v)) for k, v in grouped_spaces]
@ -330,7 +338,9 @@ class DynamicAllocationLinearProgram(object):
self.weight_spaces_sets = self._get_spaces_sets_by(criteria)
def _refresh_weight(self):
"""Create weight constraints for spaces which have same
"""Refresh weight.
Create weight constraints for spaces which have same
max constraint or for those which don't have it at all.
Lets say, second's space is equal to max of the third and fourth,
@ -356,34 +366,44 @@ class DynamicAllocationLinearProgram(object):
row = self._make_matrix_row()
weight = getattr(space, 'weight', DEFAULT_WEIGHT)
# If weight is 0, it doesn't make sense to set for such space a weight
# If weight is 0, it doesn't make sense to set for such
# space a weight
if weight == 0:
continue
space_idx = self.spaces.index(space)
for disk_idx in range(len(self.disks)):
row[disk_idx * len(self.spaces) + first_space_idx] = 1 / first_weight
row[disk_idx * len(self.spaces) + space_idx] = -1 / weight
row_i = disk_idx * len(self.spaces)
row[row_i + first_space_idx] = 1 / first_weight
row[row_i + space_idx] = -1 / weight
self.weight_equation_indexes.append(len(self.equality_constraint_matrix) - 1)
self.weight_equation_indexes.append(
len(self.equality_constraint_matrix) - 1)
self.equality_constraint_matrix.append(row)
self.equality_constraint_vector = np.append(self.equality_constraint_vector, 0)
self.equality_constraint_vector = np.append(
self.equality_constraint_vector,
0)
def _make_matrix_row(self):
return np.zeros(self.x_amount)
def _make_upper_bound_constraint_matrix(self):
"""Upper bound constraint matrix consist of upper bound
matrix and lower bound matrix witch changed sign
"""Creates upper bound constraint matrix.
Upper bound constraint matrix consist of upper bound
matrix and lower bound matrix witch changed sign.
"""
return (self.upper_bound_constraint_matrix +
[[-i for i in row] for row in self.lower_bound_constraint_matrix])
[[-i for i in row]
for row in self.lower_bound_constraint_matrix])
def _make_upper_bound_constraint_vector(self):
"""Upper bound constraint vector consist of upper bound
and lower bound, with changed sign
"""Create upper bound constraint vector.
Upper bound constraint vector consist of upper bound and
lower bound, with changed sign.
"""
return (self.upper_bound_constraint_vector +
[-i for i in self.lower_bound_constraint_vector])
@ -391,10 +411,14 @@ class DynamicAllocationLinearProgram(object):
def _convert_solution(self, solution_vector):
result = []
spaces_grouped_by_disk = list(grouper(solution_vector, len(self.spaces)))
spaces_grouped_by_disk = list(grouper(
solution_vector,
len(self.spaces)))
for disk_i in range(len(self.disks)):
disk_id = self.disks[disk_i].id
disk = {'disk_id': disk_id, 'size': self.disks[disk_i].size, 'spaces': []}
disk = {'disk_id': disk_id,
'size': self.disks[disk_i].size,
'spaces': []}
spaces_for_disk = spaces_grouped_by_disk[disk_i]
for space_i, space_size in enumerate(spaces_for_disk):
@ -410,7 +434,9 @@ class DynamicAllocationLinearProgram(object):
for d in disks:
# Initialize constraints, each row in the matrix should
# be equal to size of the disk
self.equality_constraint_vector = np.append(self.equality_constraint_vector, d.size)
self.equality_constraint_vector = np.append(
self.equality_constraint_vector,
d.size)
# Initialize the matrix
# In case of 2 spaces and 3 disks the result should be:
@ -432,7 +458,10 @@ class DynamicAllocationLinearProgram(object):
for _ in range(len(disks)):
self.equality_constraint_matrix.append(equality_matrix_row)
equality_matrix_row = shift(equality_matrix_row, len(spaces), val=0)
equality_matrix_row = shift(
equality_matrix_row,
len(spaces),
val=0)
# Size of each space should be more or equal to 0
for _ in range(self.x_amount):
@ -448,9 +477,9 @@ class DynamicAllocationLinearProgram(object):
# higher for those spaces which defined earlier
# in the list
# TODO describe why we should use special sequence
# TODO(eli): describe why we should use special sequence
# as order coefficients
coefficients = [1.0/i for i in CrossSumInequalitySequence(c_amount)]
coefficients = [1.0 / i for i in CrossSumInequalitySequence(c_amount)]
NONE_ORDER_COEFF = 1
SET_COEFF = 2
@ -484,7 +513,8 @@ class DynamicAllocationLinearProgram(object):
coefficients[c_i] += SET_COEFF
else:
# If current disk is not in the set, set it to 0
# TODO isn't it better to leave there order coefficient?
# TODO(eli): isn't it better to leave there order
# coefficient?
# coefficients[c_i] = 0
pass
else:

8
bareon_dynamic_allocator/cmd.py
View File

@ -8,7 +8,7 @@
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
@ -17,8 +17,8 @@ import sys
from oslo_config import cfg
from oslo_log import log
from bareon_dynamic_allocator import utils
from bareon_dynamic_allocator.allocators import DynamicAllocator
from bareon_dynamic_allocator import utils
from bareon_dynamic_allocator import viewer
@ -84,12 +84,12 @@ def save_result(data, output_file):
def validate_schema(schema):
# TODO should be implemented
# TODO(eli): should be implemented
return schema
def validate_hw_info(hw_info):
# TODO should be implemented
# TODO(eli): should be implemented
return hw_info

3
bareon_dynamic_allocator/errors.py
View File

@ -8,10 +8,11 @@
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class BareonDynamicAllocator(Exception):
pass

9
bareon_dynamic_allocator/parser.py
View File

@ -8,15 +8,14 @@
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import yaml
import yaql
import six
import re
import six
import yaql
def seq_iter(obj):
@ -27,7 +26,7 @@ def seq_iter(obj):
for i in xrange(len(obj)):
yield i, obj[i]
class YAQLParser(object):
engine_options = {
'yaql.limitIterators': 100,

11
bareon_dynamic_allocator/sequences.py
View File

@ -8,7 +8,7 @@
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
@ -30,7 +30,7 @@ class BaseSequence(object):
self.previous = self.current
self.current += 1
return self.current
class CrossSumInequalitySequence(BaseSequence):
"""An implementaion of a sequence from 1 to n
@ -52,7 +52,9 @@ class CrossSumInequalitySequence(BaseSequence):
class FibonacciSequence(BaseSequence):
"""Iterator over a sequence of Fibonacci numbers with n elements from 1 to n
"""Generates fibinacci sequence
Iterator over a sequence of Fibonacci numbers with n elements from 1 to n.
"""
def __init__(self, n):
super(FibonacciSequence, self).__init__(n)
@ -64,5 +66,6 @@ class FibonacciSequence(BaseSequence):
raise StopIteration
else:
self.n_current += 1
self.previous, self.current = self.current, self.current + self.previous
self.previous, self.current = \
self.current, self.current + self.previous
return self.current

5
bareon_dynamic_allocator/utils.py
View File

@ -8,7 +8,7 @@
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
@ -16,7 +16,8 @@ import yaml
def parse_yaml(path):
"""Parses yaml file
"""Parses yaml file.
:param str path: path to the file
:returns: dict or list
"""

41
bareon_dynamic_allocator/viewer.py
View File

@ -8,13 +8,13 @@
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from pprint import pprint
import svgwrite
from svgwrite import cm, mm
from svgwrite import cm
class StdoutViewer(object):
@ -36,7 +36,9 @@ class SVGViewer(object):
SPACES_X_INTERVAL = 2
STYLE = "fill:{color};stroke:black;stroke-width:5;"
def __init__(self, disks_spaces_mapping, file_path='/tmp/bareon.svg', fit=False):
def __init__(self, disks_spaces_mapping,
file_path='/tmp/bareon.svg', fit=False):
self.disks_spaces_mapping = disks_spaces_mapping
max_disk_size = max([i['size'] for i in disks_spaces_mapping])
self.width_multiplier = 450.0 / max_disk_size
@ -47,7 +49,8 @@ class SVGViewer(object):
if fit:
options = {
'preserveAspectRatio': 'none',
'viewBox': "0 0 {0} {1}".format(svg_width, svg_height) if fit else '0 0 maxY maxX'}
'viewBox': "0 0 {0} {1}".format(svg_width, svg_height)
if fit else '0 0 maxY maxX'}
self.dwg = svgwrite.Drawing(
filename=file_path,
@ -59,16 +62,26 @@ class SVGViewer(object):
self.dwg.save()
def _add_disk_with_spaces(self):
disk_g = self.dwg.add(self.dwg.g(id='disks-group', transform="translate({0}, {1})".format(30, 30)))
disk_g = self.dwg.add(self.dwg.g(
id='disks-group',
transform="translate({0}, {1})".format(30, 30)))
for disk_idx, disk_w_spaces in enumerate(self.disks_spaces_mapping):
disk_id = disk_w_spaces['disk_id']
size = disk_w_spaces['size']
disk = disk_g.add(self.dwg.g(id=disk_id, transform="translate(0, {0})".format(disk_idx * self.DISKS_INTERVAL)))
disk.add(self.dwg.text(text='{0} size={1}'.format(disk_id, size), fill="black"))
disk = disk_g.add(self.dwg.g(
id=disk_id,
transform="translate(0, {0})".format(
disk_idx * self.DISKS_INTERVAL)))
disk_rect = disk.add(self.dwg.g(transform="translate({0}, {1})".format(0, 10), id='in-{0}'.format(disk_id)))
disk.add(self.dwg.text(
text='{0} size={1}'.format(disk_id, size),
fill="black"))
disk_rect = disk.add(self.dwg.g(
transform="translate({0}, {1})".format(0, 10),
id='in-{0}'.format(disk_id)))
disk_rect.add(self.dwg.rect(
style=self.STYLE.format(color='#f5f5f5'),
ry=5,
@ -85,16 +98,22 @@ class SVGViewer(object):
rx=5,
id=space['space_id'],
insert=last_insert,
size=(self.width_multiplier * space['size'], self.SPACE_HEIGHT)))
size=(self.width_multiplier * space['size'],
self.SPACE_HEIGHT)))
last_insert[0] += self.width_multiplier * space['size']
spaces_lines = ['{0} size={1}'.format(space['space_id'], space['size']) for space in disk_w_spaces['spaces']]
spaces_lines = ['{0} size={1}'.format(space['space_id'],
space['size'])
for space in disk_w_spaces['spaces']]
last_insert[0] = self.width_multiplier * disk_w_spaces['size']
last_insert[0] += 10
last_insert[1] += 20
for space_idx, space_line in enumerate(spaces_lines):
palette = self.PALETTE[space_idx % len(self.PALETTE)]
disk_rect.add(self.dwg.text(insert=last_insert, text=space_line, fill=palette))
disk_rect.add(self.dwg.text(
insert=last_insert,
text=space_line,
fill=palette))
last_insert[1] += 20

44
doc_generate_static.py
View File

@ -8,47 +8,65 @@
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
import os
import six
from glob import glob
from bareon_dynamic_allocator.allocators import DynamicAllocator
from bareon_dynamic_allocator import utils
from bareon_dynamic_allocator import viewer
from bareon_dynamic_allocator.allocators import DynamicAllocator
doc_schemas_path = os.path.join(os.path.dirname(os.path.realpath(__file__)) , 'doc', 'source', 'schemas')
doc_schemas_rst_path = os.path.join(os.path.dirname(os.path.realpath(__file__)) , 'doc', 'source', 'examples.rst')
doc_schemas_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'doc',
'source',
'schemas')
doc_schemas_rst_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'doc',
'source',
'examples.rst')
def generate_svg_files():
result = {}
for dynamic_schema_path in sorted(glob(os.path.join(doc_schemas_path, '*_ds.yaml'))):
print 'Read file {0}'.format(dynamic_schema_path)
for dynamic_schema_path in sorted(
glob(os.path.join(doc_schemas_path, '*_ds.yaml'))):
print('Read file {0}'.format(dynamic_schema_path))
dynamic_schema = utils.parse_yaml(dynamic_schema_path)
dynamic_schema_file_name = os.path.basename(dynamic_schema_path)
dynamic_schema_name = os.path.splitext(dynamic_schema_file_name)[0]
for hw_info_path in sorted(glob(os.path.join(doc_schemas_path, '*_disk.yaml'))):
print 'Read file {0}'.format(hw_info_path)
for hw_info_path in sorted(
glob(os.path.join(doc_schemas_path, '*_disk.yaml'))):
print('Read file {0}'.format(hw_info_path))
hw_info_file_name = os.path.basename(hw_info_path)
hw_info_name = os.path.splitext(hw_info_file_name)[0]
hw_info = utils.parse_yaml(hw_info_path)
static_schema = DynamicAllocator(hw_info, dynamic_schema).generate_static()
static_schema = DynamicAllocator(
hw_info,
dynamic_schema).generate_static()
static_schema_name = '{0}_{1}.svg'.format(dynamic_schema_name, hw_info_name)
static_schema_name = '{0}_{1}.svg'.format(
dynamic_schema_name,
hw_info_name)
result[static_schema_name[:-4]] = {
'dynamic_schema': os.path.join('schemas', dynamic_schema_file_name),
'dynamic_schema': os.path.join('schemas',
dynamic_schema_file_name),
'hw_info': os.path.join('schemas', hw_info_file_name),
'image': os.path.join('schemas', static_schema_name)}
viewer.SVGViewer(static_schema, file_path=os.path.join(doc_schemas_path, static_schema_name), fit=True).show_me()
viewer.SVGViewer(static_schema,
file_path=os.path.join(doc_schemas_path,
static_schema_name),
fit=True).show_me()
rst_doc = """
===================

15
lab_mip.py
View File

@ -12,8 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from pulp import *
from pulp import * # flake8: noqa
import sys
x = []
@ -52,7 +51,7 @@ prob += x1 + x2 <= 100, 'First disk'
prob += x3 + x4 <= 100, 'Second disk'
prob += x5 + x6 <= 100, 'Third disk'
prob += y1 + y3 + y5 == 2, 'Replication factor'
prob += y1 + y3 + y5 == 1, 'Replication factor'
prob += x2 + x4 + x6 >= 10, 'Second min size'
@ -102,6 +101,7 @@ for i, x_ in enumerate(x):
# solve the problem
status = prob.solve(GLPK(msg=1))
def print_vector(vector, prefix, n=2):
for i, v in enumerate(vector):
@ -111,10 +111,11 @@ def print_vector(vector, prefix, n=2):
else:
sys.stdout.write('\n')
print
print()
print_vector(x, 'x')
print
print()
print_vector(y, 'y')
print
print()
print_vector(z, 'z', n=3)
print
print()