Updating doc strings for core pieces of Syntribos

This PR adds docstrings to a number of important components of
Syntribos, and adds this documentation to our Sphinx doc structure. It
also removes copyrights from __init__.py files with no other content,
in line with OpenStack style guidelines.

Set 2: Fixed PEP8 failure.

Change-Id: Ic57b31f451ec3ecf7f5b308da4544f808c9c9a5d
Implements: blueprint docstring-add-to-framework
This commit is contained in:
Charles Neill 2016-04-04 15:42:21 -05:00
parent d9d6e5ed4e
commit 9eca39e127
20 changed files with 239 additions and 164 deletions

View File

@ -15,14 +15,16 @@
import os
import sys
sys.path.insert(0, os.path.abspath('../..'))
sys.path.insert(0, os.path.abspath('../../'))
sys.path.insert(0, os.path.abspath('../../syntribos'))
# -- General configuration ----------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'sphinx.ext.autodoc',
# 'sphinx.ext.intersphinx',
'sphinx.ext.intersphinx',
'oslosphinx'
]
@ -75,3 +77,8 @@ latex_documents = [
# Example configuration for intersphinx: refer to the Python standard library.
# intersphinx_mapping = {'http://docs.python.org/': None}
autodoc_mock_imports = [
'cafe',
'cafe.engine.http.client',
'cafe.drivers.unittest.arguments'
]

View File

@ -27,9 +27,16 @@ Index
running
logging
test.anatomy
unittests
contributing
For Developers
--------------
.. toctree::
:maxdepth: 1
contributing
code-docs
unittests
Project information
-------------------

View File

@ -1,15 +0,0 @@
"""
Copyright 2015 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -22,11 +22,21 @@ import cafe.drivers.unittest.arguments
class InputType(object):
"""Reads a file/directory, or stdin, to collect request templates."""
def __init__(self, mode, bufsize):
self._mode = mode
self._bufsize = bufsize
def __call__(self, string):
"""Yield the name and contents of the 'input' file(s)
:param str string: the value supplied as the 'input' argument
:rtype: tuple
:returns: (file name, file contents)
"""
if string == '-':
fp = sys.stdin
yield fp.name, fp.read()
@ -51,6 +61,9 @@ class InputType(object):
class SyntribosCLI(argparse.ArgumentParser):
"""Class for parsing Syntribos command-line arguments."""
def __init__(self, *args, **kwargs):
super(SyntribosCLI, self).__init__(*args, **kwargs)
self._add_args()

View File

@ -1,15 +0,0 @@
"""
Copyright 2015 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -22,8 +22,18 @@ _iterators = {}
class RequestHelperMixin(object):
"""Class that helps with fuzzing requests."""
@classmethod
def _run_iters(cls, data, action_field):
"""Recursively fuzz variables in `data` and its children
:param data: The request data to be modified
:param action_field: The name of the field to be replaced
:returns: object or string with action_field fuzzed
:rtype: `dict` OR `str` OR :class:`ElementTree.Element`
"""
if isinstance(data, dict):
return cls._run_iters_dict(data, action_field)
elif isinstance(data, ElementTree.Element):
@ -35,6 +45,7 @@ class RequestHelperMixin(object):
@classmethod
def _run_iters_dict(cls, dic, action_field=""):
"""Run fuzz iterators for a dict type."""
for key, val in dic.iteritems():
dic[key] = val = cls._replace_iter(val)
if isinstance(key, basestring):
@ -50,6 +61,7 @@ class RequestHelperMixin(object):
@classmethod
def _run_iters_list(cls, val, action_field=""):
"""Run fuzz iterators for a list type."""
for i, v in enumerate(val):
if isinstance(v, basestring):
val[i] = v = cls._replace_iter(v).replace(action_field, "")
@ -60,6 +72,7 @@ class RequestHelperMixin(object):
@classmethod
def _run_iters_xml(cls, ele, action_field=""):
"""Run fuzz iterators for an XML element type."""
if isinstance(ele.text, basestring):
ele.text = cls._replace_iter(ele.text).replace(action_field, "")
cls._run_iters_dict(ele.attrib, action_field)
@ -69,6 +82,7 @@ class RequestHelperMixin(object):
@staticmethod
def _string_data(data):
"""Replace various objects types with string representations."""
if isinstance(data, dict):
return json.dumps(data)
elif isinstance(data, ElementTree.Element):
@ -78,6 +92,7 @@ class RequestHelperMixin(object):
@staticmethod
def _replace_iter(string):
"""Fuzz a string."""
if not isinstance(string, basestring):
return string
for k, v in _iterators.items():
@ -86,11 +101,11 @@ class RequestHelperMixin(object):
return string
def prepare_request(self):
"""prepare a request
"""Prepare a request for sending off
it should be noted this function does not make a request copy
It should be noted this function does not make a request copy,
destroying iterators in request. A copy should be made if making
multiple requests
multiple requests.
"""
self.data = self._run_iters(self.data, self.action_field)
self.headers = self._run_iters(self.headers, self.action_field)
@ -98,6 +113,11 @@ class RequestHelperMixin(object):
self.data = self._string_data(self.data)
def get_prepared_copy(self):
"""Create a copy of `self`, and prepare it for use by a fuzzer
:returns: Copy of request object that has been prepared for sending
:rtype: :class:`RequestHelperMixin`
"""
local_copy = copy.deepcopy(self)
local_copy.prepare_request()
return local_copy
@ -107,6 +127,9 @@ class RequestHelperMixin(object):
class RequestObject(object):
"""An object that holds information about an HTTP request."""
def __init__(
self, method, url, action_field=None, headers=None, params=None,
data=None):

View File

@ -32,6 +32,15 @@ class RequestCreator(object):
@classmethod
def create_request(cls, string, endpoint):
"""Parse the HTTP request template into its components
:param str string: HTTP request template
:param str endpoint: URL of the target to be tested
:rtype: :class:`syntribos.clients.http.models.RequestObject`
:returns: RequestObject with method, url, params, etc. for use by
runner
"""
string = cls.call_external_functions(string)
action_field = str(uuid.uuid4()).replace("-", "")
string = string.replace(cls.ACTION_FIELD, action_field)
@ -50,6 +59,14 @@ class RequestCreator(object):
@classmethod
def _parse_url_line(cls, line, endpoint):
"""Split first line of an HTTP request into its components
:param str line: the first line of the HTTP request
:param str endpoint: the full URL of the endpoint to test
:rtype: tuple
:returns: HTTP method, URL, request parameters, HTTP version
"""
params = {}
method, url, version = line.split()
url = url.split("?", 1)
@ -66,6 +83,13 @@ class RequestCreator(object):
@classmethod
def _parse_headers(cls, lines):
"""Find and return headers in HTTP request
:param str lines: All but the first line of the HTTP request (list)
:rtype: dict
:returns: headers as key:value pairs
"""
headers = {}
for line in lines:
key, value = line.split(":", 1)
@ -74,6 +98,12 @@ class RequestCreator(object):
@classmethod
def _parse_data(cls, lines):
"""Parse the body of the HTTP request (e.g. POST variables)
:param list lines: lines of the HTTP body
:returns: object representation of body data (JSON or XML)
"""
data = "\n".join(lines).strip()
if not data:
return ""
@ -88,6 +118,14 @@ class RequestCreator(object):
@classmethod
def call_external_functions(cls, string):
"""Parse external function calls in the body of request templates
:param str string: full HTTP request template as a string
:rtype: str
:returns: the request, with EXTERNAL calls filled in with their values
or UUIDs
"""
if not isinstance(string, basestring):
return string

View File

@ -18,13 +18,17 @@ import cafe.engine.models.data_interfaces as data_interfaces
class MainConfig(data_interfaces.ConfigSectionInterface):
'''Reads in configuration data from config file.'''
"""Reads in configuration data from config file."""
SECTION_NAME = "syntribos"
@property
def endpoint(self):
"""The target host to be tested."""
return self.get("endpoint")
@property
def version(self):
"""???"""
return self.get("version")

View File

@ -1,15 +0,0 @@
"""
Copyright 2015 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -1,15 +0,0 @@
"""
Copyright 2015 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -1,15 +0,0 @@
"""
Copyright 2015 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -1,15 +0,0 @@
"""
Copyright 2015 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -14,13 +14,15 @@ under the License.
class Issue(object):
"""Object that encapsulates security vulnerability
"""Object that encapsulates a security vulnerability
This object is designed to hold the metadata associated with
an vulnerability, as well as the requests and responses that
a vulnerability, as well as the requests and responses that
caused the vulnerability to be flagged. Furthermore, holds the
assertions actually run by the test case
"""
def __init__(self, severity, test="", text="", confidence="",
request=None, response=None):
self.test = test
@ -32,7 +34,11 @@ class Issue(object):
self.failure = False
def as_dict(self):
'''Convert the issue to a dict of values for outputting.'''
"""Convert the issue to a dict of values for outputting.
:rtype: `dict`
:returns: dictionary of issue data
"""
out = {
'test_name': self.test,
'issue_severity': self.severity,
@ -44,7 +50,13 @@ class Issue(object):
return out
def request_as_dict(self, req):
'''Convert the request object to a dict of values for outputting.'''
"""Convert the request object to a dict of values for outputting.
:param req: The request object
:type req: (TODO)
:rtype: `dict`
:returns: dictionary of HTTP request data
"""
return {
'url': req.path_url,
'method': req.method,
@ -54,7 +66,13 @@ class Issue(object):
}
def response_as_dict(self, res):
'''Convert the response object to a dict of values for outputting.'''
"""Convert the response object to a dict of values for outputting.
:param res: The result object
:type res: (TODO)
:rtype: `dict`
:returns: dictionary of HTTP response data
"""
return {
'status_code': res.status_code,
'reason': res.reason,

View File

@ -19,21 +19,24 @@ from syntribos.formatters.json_formatter import JSONFormatter
class IssueTestResult(unittest.TextTestResult):
"""Custom unnittest results holder class
A test result class that can return issues raised by tests
to the Syntribos runner
This class aggregates :class:`syntribos.issue.Issue` objects from all the
tests as they run
"""
aggregated_failures = {}
pruned_failures = []
def addFailure(self, test, err):
"""Adds failed issues to data structures
"""Adds issues to data structures
Appends failed issues to the result's list of failures, as well as
to a dict of {url:
method:
test_name: issue} structure.
Appends issues to the result's list of failures, as well as
to a dict of {url: {method: {test_name: issue}}} structure.
:param test: The test that has failed
:type test: :class:`syntribos.tests.base.BaseTestCase`
:param tuple err: Tuple of format ``(type, value, traceback)``
"""
self.failures.append((test, test.failures))
for issue in test.failures:
@ -58,10 +61,20 @@ class IssueTestResult(unittest.TextTestResult):
sys.stdout.flush()
def addError(self, test, err):
"""Duplicates parent class addError functionality."""
"""Duplicates parent class addError functionality.
:param test: The test that encountered an error
:type test: :class:`syntribos.tests.base.BaseTestCase`
:param err:
:type tuple: Tuple of format ``(type, value, traceback)``
"""
super(IssueTestResult, self).addError(test, err)
def printErrors(self, output_format):
"""Print out each :class:`syntribos.issue.Issue` that was encountered
:param str output_format: Either "json" or "xml"
"""
formatter_types = {
"json": JSONFormatter(self)
}
@ -72,5 +85,6 @@ class IssueTestResult(unittest.TextTestResult):
formatter.report()
def stopTestRun(self):
"""Print errors when the test run is complete."""
super(IssueTestResult, self).stopTestRun()
self.printErrors()

View File

@ -44,11 +44,16 @@ class Runner(object):
@classmethod
def print_tests(cls):
"""Print out all the tests that will be run."""
for name, test in cls.get_tests():
print(name)
@classmethod
def load_modules(cls, package):
"""Imports all tests (:mod:`syntribos.tests`)
:param package: a package of tests for pkgutil to load
"""
if not os.environ.get("CAFE_CONFIG_FILE_PATH"):
os.environ["CAFE_CONFIG_FILE_PATH"] = "./"
for importer, modname, ispkg in pkgutil.walk_packages(
@ -59,6 +64,13 @@ class Runner(object):
@classmethod
def get_tests(cls, test_types=None):
"""Yields relevant tests based on test type (from ```syntribos.arguments```)
:param list test_types: Test types to be run
:rtype: tuple
:returns: (test type (str), ```syntribos.tests.base.TestType```)
"""
cls.load_modules(tests)
test_types = test_types or [""]
for k, v in sorted(syntribos.tests.base.test_table.items()):
@ -94,6 +106,7 @@ class Runner(object):
@staticmethod
def print_log():
"""Print the path to the log folder for this run."""
test_log = os.environ.get("CAFE_TEST_LOG_PATH")
if test_log:
print("=" * 70)
@ -113,6 +126,7 @@ class Runner(object):
"""
args, unknown = syntribos.arguments.SyntribosCLI(
usage=usage).parse_known_args()
sys.stdout.write("TYPE: {0}".format(type(args)))
test_env_manager = TestEnvManager(
"", args.config, test_repo_package_name="os")
test_env_manager.finalize()
@ -145,7 +159,16 @@ class Runner(object):
@classmethod
def run_test(cls, test, result, dry_run=False):
"""Create a new test suite, add a test, and run it
:param test: The test to add to the suite
:param result: The result object to append to
:type result: :class:`syntribos.result.IssueTestResult`
:param bool dry_run: (OPTIONAL) Only print out test names
"""
suite = cafe.drivers.unittest.suite.OpenCafeUnittestTestSuite()
suite = unittest.TestSuite()
suite.addTest(test("run_test"))
if dry_run:
for test in suite:
@ -155,12 +178,20 @@ class Runner(object):
@classmethod
def set_env(cls):
"""Set environment variables for this run."""
config = syntribos.config.MainConfig()
os.environ["SYNTRIBOS_ENDPOINT"] = config.endpoint
@classmethod
def print_result(cls, result, start_time, args):
"""Prints results summerized."""
"""Prints test summary/stats (e.g. # failures) to stdout
:param result: Global result object with all issues/etc.
:type result: :class:`syntribos.result.IssueTestResult`
:param float start_time: Time this run started
:param args: Parsed CLI arguments
:type args: ``argparse.Namespace``
"""
result.printErrors(args.output_format)
run_time = time.time() - start_time
tests = result.testsRun
@ -180,6 +211,7 @@ class Runner(object):
def entry_point():
"""Start runner. Need this so we can point to it in ``setup.cfg``."""
Runner.run()
return 0

View File

@ -1,15 +0,0 @@
"""
Copyright 2015 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -1,15 +0,0 @@
"""
Copyright 2015 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

View File

@ -24,21 +24,28 @@ from syntribos.issue import Issue
ALLOWED_CHARS = "().-_{0}{1}".format(t_string.ascii_letters, t_string.digits)
'''test_table is the master list of tests to be run by the runner'''
"""test_table is the master list of tests to be run by the runner"""
test_table = {}
def replace_invalid_characters(string, new_char="_"):
"""Replace invalid characters
"""Replace invalid characters in test names
This function corrects `string` so the following is true.
This functions corrects string so the following is true
Identifiers (also referred to as names) are described by the
following lexical definitions:
identifier ::= (letter|"_") (letter | digit | "_")*
letter ::= lowercase | uppercase
lowercase ::= "a"..."z"
uppercase ::= "A"..."Z"
digit ::= "0"..."9"
| ``identifier ::= (letter|"_") (letter | digit | "_")*``
| ``letter ::= lowercase | uppercase``
| ``lowercase ::= "a"..."z"``
| ``uppercase ::= "A"..."Z"``
| ``digit ::= "0"..."9"``
:param str string: Test name
:param str new_char: The character to replace invalid characters with
:returns: The test name, with invalid characters replaced with `new_char`
:rtype: str
"""
if not string:
return string
@ -50,6 +57,9 @@ def replace_invalid_characters(string, new_char="_"):
class TestType(type):
"""This is the metaclass for each class extending :class:`BaseTestCase`."""
def __new__(cls, cls_name, cls_parents, cls_attr):
new_class = super(TestType, cls).__new__(
cls, cls_name, cls_parents, cls_attr)
@ -62,23 +72,35 @@ class TestType(type):
@six.add_metaclass(TestType)
class BaseTestCase(cafe.drivers.unittest.fixtures.BaseTestFixture):
"""Base Class
Base for building new tests
"""Base class for building new tests
:attribute test_name: A name like ``XML_EXTERNAL_ENTITY_BODY``, containing
the test type and the portion of the request template being tested
"""
test_name = None
@classmethod
def get_test_cases(cls, filename, file_content):
"""Not sure what the point of this is.
TODO: FIGURE THIS OUT
"""
yield cls
@classmethod
def extend_class(cls, new_name, kwargs):
'''Creates an extension for the class
"""Creates an extension for the class
Each TestCase class created is added to the test_table, which is then
Each TestCase class created is added to the `test_table`, which is then
read in by the test runner as the master list of tests to be run.
'''
:param str new_name: Name of new class to be created
:param dict kwargs: Keyword arguments to pass to the new class
:rtype: class
:returns: A TestCase class extending :class:`BaseTestCase`
"""
new_name = replace_invalid_characters(new_name)
if not isinstance(kwargs, dict):
raise Exception("kwargs must be a dictionary")
@ -97,9 +119,13 @@ class BaseTestCase(cafe.drivers.unittest.fixtures.BaseTestFixture):
def register_issue(self, issue=None):
"""Adds an issue to the test's list of issues
Creates a new issue object, and associates the test's request
and response to it. In addition, adds the issue to the test's
list of issues.
Creates a new :class:`syntribos.issue.Issue` object, and associates the
test's request and response to it. In addition, adds the issue to the
test's list of issues.
:param Issue issue: (OPTIONAL) issue object to update
:returns: new issue object with request and response associated
:rtype: Issue
"""
if not issue:
@ -110,3 +136,12 @@ class BaseTestCase(cafe.drivers.unittest.fixtures.BaseTestFixture):
self.failures.append(issue)
return issue
def test_issues(self):
"""Run assertions for each test registered in test_case."""
for issue in self.issues:
try:
issue.run_tests()
except AssertionError:
self.failures.append(issue)
raise

View File

@ -80,11 +80,12 @@ class BaseFuzzTestCase(base.BaseTestCase):
@classmethod
def data_driven_failure_cases(cls):
'''Checks if response contains known bad strings
"""Checks if response contains known bad strings
:returns: a list of strings that show up in the response that are also
defined in cls.failure_strings.
'''
failed_strings = []
"""
failed_strings = []
if cls.failure_keys is None:
return []
@ -95,12 +96,12 @@ class BaseFuzzTestCase(base.BaseTestCase):
@classmethod
def data_driven_pass_cases(cls):
'''Checks if response contains expected strings
"""Checks if response contains expected strings
:returns: a list of assertions that fail if the response doesn't
contain a string defined in cls.success_keys as a string expected in
the response.
'''
"""
if cls.success_keys is None:
return True
for s in cls.success_keys:
@ -158,7 +159,7 @@ class BaseFuzzTestCase(base.BaseTestCase):
"injection attacks")
.format(self.config.percent)
)
)
)
def test_case(self):
"""Performs the test
@ -178,7 +179,7 @@ class BaseFuzzTestCase(base.BaseTestCase):
For each string returned by cls._get_strings(), yield a TestCase class
for the string as an extension to the current TestCase class. Every
string used as a fuzz test payload entails the generation of a new
subclass for each parameter fuzzed. See base.extend_class().
subclass for each parameter fuzzed. See :func:`base.extend_class`.
"""
# maybe move this block to base.py
request_obj = syntribos.tests.fuzz.datagen.FuzzParser.create_request(

View File

@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
import re
import sys
from xml.etree import ElementTree
from syntribos.clients.http.models import RequestHelperMixin
@ -145,6 +146,7 @@ class FuzzMixin(object):
class FuzzRequest(RequestObject, FuzzMixin, RequestHelperMixin):
def fuzz_request(self, strings, fuzz_type, name_prefix):
"""Creates the fuzzed request object
@ -154,6 +156,7 @@ class FuzzRequest(RequestObject, FuzzMixin, RequestHelperMixin):
for name, data in self._fuzz_data(
strings, getattr(self, fuzz_type), self.action_field,
name_prefix):
sys.stdout.write("Name: {0}\nData: {1}\n".format(name, data))
request_copy = self.get_copy()
setattr(request_copy, fuzz_type, data)
request_copy.prepare_request(fuzz_type)