Add tox/requirements settings to pass gate job

Plus, we need liberasurecode version handling for a few place because
some tests/engine itself is broken with a lack of backword
compatibility.

Closes-Bug: #1586220

Change-Id: I72adaefa10875a73e3e5304eb40fe5d9f6d2598a

bindep.txt

@@ -0,0 +1,7 @@
+# This is a cross-platform list tracking distribution packages needed by tests;
+# see http://docs.openstack.org/infra/bindep/ for additional information.
+
+build-essential [platform:dpkg]
+gcc [platform:rpm]
+liberasurecode-dev [platform:dpkg]
+liberasurecode-devel [platform:rpm]

doc/source/conf.py

@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+#
+# PyECLib documentation build configuration file, created by
+# sphinx-quickstart on Thu Aug 11 19:59:50 2016.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'PyECLib'
+copyright = u'2016, Kevin Greenan, Tushar Gohad, Kota Tsuyuzaki'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '1.2.1'
+# The full version, including alpha/beta/rc tags.
+release = '1.2.1'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'PyECLibdoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+  ('index', 'PyECLib.tex', u'PyECLib Documentation',
+   u'Kevin Greenan, Tushar Gohad, Kota Tsuyuzaki', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'pyeclib', u'PyECLib Documentation',
+     [u'Kevin Greenan, Tushar Gohad, Kota Tsuyuzaki'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('index', 'PyECLib', u'PyECLib Documentation',
+   u'Kevin Greenan, Tushar Gohad, Kota Tsuyuzaki', 'PyECLib', 'One line description of project.',
+   'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'

doc/source/index.rst

@@ -0,0 +1,22 @@
+.. PyECLib documentation master file, created by
+   sphinx-quickstart on Thu Aug 11 19:59:50 2016.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to PyECLib's documentation!
+===================================
+
+Contents:
+
+.. toctree::
+   :maxdepth: 2
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

pyeclib/ec_iface.py

@@ -26,7 +26,35 @@ from .enum import Enum
 from .enum import unique
 from .utils import create_instance
 from .utils import positive_int_value
-from pyeclib_c import check_backend_available
+from pyeclib_c import get_liberasurecode_version
+
+
+def check_backend_available(backend_name):
+    try:
+        from pyeclib_c import check_backend_available
+
+        if backend_name.startswith('flat_xor_hd'):
+            int_type = PyECLib_EC_Types.get_by_name('flat_xor_hd')
+        else:
+            int_type = PyECLib_EC_Types.get_by_name(backend_name)
+        if not int_type:
+            return False
+        return check_backend_available(int_type.value)
+    except ImportError:
+        # check_backend_available has been supported since
+        # liberasurecode>=1.2.0 so we need to define the func for older
+        # liberasurecode version
+
+        # select available k, m values
+        if backend_name.startswith('flat_xor_hd'):
+            k, m = (10, 5)
+        else:
+            k, m = (10, 4)
+        try:
+            driver = ECDriver(ec_type=backend_name, k=k, m=m)
+        except ECDriverError:
+            return False
+        return True
 
 
 def PyECLibVersion(z, y, x):
@@ -477,15 +505,22 @@ ALL_EC_TYPES = [
 def _PyECLibValidECTypes():
     available_ec_types = []
     for _type in ALL_EC_TYPES:
-        if _type.startswith('flat_xor_hd'):
-            int_type = PyECLib_EC_Types.get_by_name('flat_xor_hd')
-        else:
-            int_type = PyECLib_EC_Types.get_by_name(_type)
-        if not int_type:
-            continue
-        if check_backend_available(int_type.value):
+        if check_backend_available(_type):
             available_ec_types.append(_type)
     return available_ec_types
 
 
 VALID_EC_TYPES = _PyECLibValidECTypes()
+
+
+def _liberasurecode_version():
+    version_int = get_liberasurecode_version()
+    version_hex_str = hex(version_int)
+    version_hex_str = version_hex_str.lstrip('0x')
+    major = str(int(version_hex_str[-6:-4]))
+    minor = str(int(version_hex_str[-4:-2]))
+    rev = str(int(version_hex_str[-2:]))
+    version_str = '.'.join([major, minor, rev])
+    return version_str
+
+LIBERASURECODE_VERSION = _liberasurecode_version()

src/c/pyeclib_c/pyeclib_c.c

@@ -85,6 +85,7 @@ static PyObject * pyeclib_c_reconstruct(PyObject *self, PyObject *args);
 static PyObject * pyeclib_c_decode(PyObject *self, PyObject *args);
 static PyObject * pyeclib_c_get_metadata(PyObject *self, PyObject *args);
 static PyObject * pyeclib_c_check_metadata(PyObject *self, PyObject *args);
+static PyObject * pyeclib_c_liberasurecode_version(PyObject *self, PyObject *args);
 
 static PyObject *import_class(const char *module, const char *cls)
 {
@@ -1189,6 +1190,11 @@ pyeclib_c_check_backend_available(PyObject *self, PyObject *args)
     Py_RETURN_FALSE;
 }
 
+static PyObject*
+pyeclib_c_liberasurecode_version(PyObject *self, PyObject *args){
+    return PyInt_FromLong(LIBERASURECODE_VERSION);
+}
+
 static PyMethodDef PyECLibMethods[] = {
     {"init",  pyeclib_c_init, METH_VARARGS, "Initialize a new erasure encoder/decoder"},
     {"encode",  pyeclib_c_encode, METH_VARARGS, "Create parity using source data"},
@@ -1198,7 +1204,10 @@ static PyMethodDef PyECLibMethods[] = {
     {"get_segment_info", pyeclib_c_get_segment_info, METH_VARARGS, "Return segment and fragment size information needed when encoding a segmented stream"},
     {"get_metadata", pyeclib_c_get_metadata, METH_VARARGS, "Get the integrity checking metadata for a fragment"},
     {"check_metadata", pyeclib_c_check_metadata, METH_VARARGS, "Check the integrity checking metadata for a set of fragments"},
+    {"get_liberasurecode_version", pyeclib_c_liberasurecode_version, METH_NOARGS, "Get libersaurecode version in use"},
+#if ( LIBERASURECODE_VERSION >= _VERSION(1,2,0) )
     {"check_backend_available", pyeclib_c_check_backend_available, METH_VARARGS, "Check if a backend is available"},
+#endif
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 

test-requirements.txt

@@ -0,0 +1,6 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+nose
+sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3  # BSD

test/test_pyeclib_api.py

@@ -26,6 +26,9 @@ from string import ascii_letters, ascii_uppercase, digits
 import sys
 import tempfile
 import unittest
+
+from distutils.version import StrictVersion
+
 from pyeclib.ec_iface import ECBackendNotSupported
 from pyeclib.ec_iface import ECDriver
 from pyeclib.ec_iface import ECDriverError
@@ -34,6 +37,7 @@ from pyeclib.ec_iface import ECInvalidFragmentMetadata
 from pyeclib.ec_iface import PyECLib_EC_Types
 from pyeclib.ec_iface import ALL_EC_TYPES
 from pyeclib.ec_iface import VALID_EC_TYPES
+from pyeclib.ec_iface import LIBERASURECODE_VERSION
 
 if sys.version < '3':
     def b2i(b):
@@ -531,6 +535,11 @@ class TestPyECLibDriver(unittest.TestCase):
                       (first_fragment_to_corrupt + i) % len(fragments) for i in range(num_to_corrupt)
                     ]
 
+                    if StrictVersion(LIBERASURECODE_VERSION) < \
+                            StrictVersion('1.2.0'):
+                        # if liberasurecode is older than the version supports
+                        # fragment integrity check, skip following test
+                        continue
                     i = 0
                     for fragment in fragments:
                       if i in fragments_to_corrupt:

tox.ini

@@ -2,8 +2,8 @@
 envlist = py27,py34,pep8
 
 [testenv]
-deps=
-    nose
+deps =
+  -r{toxinidir}/test-requirements.txt
 commands=
     nosetests test/
 
@@ -12,3 +12,9 @@ deps=
     pep8
 commands=
     pep8 pyeclib/ setup.py
+
+[testenv:venv]
+commands = {posargs}
+
+[testenv:docs]
+commands = python setup.py build_sphinx