openstack
/
syntribos
Code Issues Proposed changes

Test logic is no longer written in terms of assertions 

Changes to the framework are only found in runner.py and base.py.

Tests are now no longer written as lists of assertions, which before meant that
it was impossible to access any variables used in the test itself after the
test was over. Instead, the register_issue method now adds the issue to
cls.failures, and will throw an AssertionError if there are any failures
reported to the test runner. This will allow us to do things like put the
failure string into the Issue description.

Change-Id: Ic3ca2ec48a6e1d99d56e605a2e0d0dc89158bc72
Implements: bp/framework-issue-creation
This commit is contained in:
michael.dong@rackspace.com 2016-03-29 17:51:05 -05:00
parent f39f59c294
commit 3fff915563
7 changed files with 108 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
syntribos/runner.py
View File

@ -146,7 +146,7 @@ class Runner(object):
@classmethod
def run_test(cls, test, result, dry_run=False):
suite = cafe.drivers.unittest.suite.OpenCafeUnittestTestSuite()
suite.addTest(test("test_case"))
suite.addTest(test("run_test"))
if dry_run:
for test in suite:
print(test)

18
syntribos/tests/base.py
View File

@ -86,6 +86,11 @@ class BaseTestCase(cafe.drivers.unittest.fixtures.BaseTestFixture):
new_cls.__module__ = cls.__module__
return new_cls
def run_test(self):
self.test_case()
if self.failures:
raise AssertionError
def test_case(self):
pass
@ -94,7 +99,7 @@ class BaseTestCase(cafe.drivers.unittest.fixtures.BaseTestFixture):
Creates a new issue object, and associates the test's request
and response to it. In addition, adds the issue to the test's
list of issues
list of issues.
"""
if not issue:
@ -102,15 +107,6 @@ class BaseTestCase(cafe.drivers.unittest.fixtures.BaseTestFixture):
issue.request = self.resp.request
issue.response = self.resp
self.issues.append(issue)
self.failures.append(issue)
return issue
def test_issues(self):
'''run assertions for each test registered in test_case.'''
for issue in self.issues:
try:
issue.run_tests()
except AssertionError:
self.failures.append(issue)
raise

82
syntribos/tests/fuzz/base_fuzz.py
View File

@ -38,6 +38,8 @@ class BaseFuzzTestCase(base.BaseTestCase):
Compares the length of a fuzzed response with a response to the
baseline request. If the response is longer than expected, returns
false
:returns: boolean - whether the response is longer than expected
"""
if getattr(cls, "init_response", False) is False:
raise NotImplemented
@ -80,25 +82,24 @@ class BaseFuzzTestCase(base.BaseTestCase):
def data_driven_failure_cases(cls):
'''Checks if response contains known bad strings
Returns a list of assertions that fail if the response contains
any string defined in cls.failure_keys as a string indicating a
failure to some sort of attack.
:returns: a list of strings that show up in the response that are also
defined in cls.failure_strings.
'''
failure_assertions = []
failed_strings = []
if cls.failure_keys is None:
return []
for line in cls.failure_keys:
failure_assertions.append((cls.assertNotIn,
line, cls.resp.content))
return failure_assertions
if line in cls.resp.content:
failed_strings.append(line)
return failed_strings
@classmethod
def data_driven_pass_cases(cls):
'''Checks if response contains expected strings
Returns a list of assertions that fail if the response doesn't contain
a string defined in cls.success_keys as a string expected in the
response.
:returns: a list of assertions that fail if the response doesn't
contain a string defined in cls.success_keys as a string expected in
the response.
'''
if cls.success_keys is None:
return True
@ -111,7 +112,6 @@ class BaseFuzzTestCase(base.BaseTestCase):
def setUpClass(cls):
"""being used as a setup test not."""
super(BaseFuzzTestCase, cls).setUpClass()
cls.issues = []
cls.failures = []
cls.resp = cls.client.request(
method=cls.request.method, url=cls.request.url,
@ -122,47 +122,51 @@ class BaseFuzzTestCase(base.BaseTestCase):
def tearDownClass(cls):
super(BaseFuzzTestCase, cls).tearDownClass()
def register_default_tests(self):
"""Registers default issues
def test_default_issues(self):
"""Tests for some default issues
These issues are not specific to any test type, and can be raised as a
result of many different types of attacks. Therefore, they're defined
separately from the test_case method so that they are not overwritten
by test cases that inherit from BaseFuzzTestCase.
Any extension to this class should call
self.test_default_issues() in order to test for the Issues
defined here
"""
self.register_issue(
Issue(test="500_errors",
severity="Low",
confidence="High",
text=("This request returns an error with status code >= 500"
"which might indicate some server-side fault that"
"could lead to further vulnerabilities"
),
assertions=[(self.assertTrue, self.resp.status_code < 500)])
)
self.register_issue(
Issue(test="length_diff",
severity="Low",
confidence="Low",
text=("The difference in length between the response to the"
"baseline request and the request returned when"
"sending an attack string exceeds {0} percent, which"
"could indicate a vulnerability to injection attacks")
.format(self.config.percent),
assertions=[(self.assertTrue, self.validate_length())]))
if self.resp.status_code >= 500:
self.register_issue(
Issue(test="500_errors",
severity="Low",
confidence="High",
text=("This request returns an error with status code"
"{0}, which might indicate some server-side fault"
"that could lead to further vulnerabilities"
).format(self.resp.status_code)
)
)
if not self.validate_length():
self.register_issue(
Issue(test="length_diff",
severity="Low",
confidence="Low",
text=("The difference in length between the response to"
"the baseline request and the request returned"
"when sending an attack string exceeds {0}"
"percent, which could indicate a vulnerability to"
"injection attacks")
.format(self.config.percent)
)
)
def test_case(self):
"""Performs the test
The test runner will call test_case on every TestCase class, and will
report any AssertionError raised by this method to the results.
Any extension to this class should call
super(type(self), self).test_case in order to test for the Issues
defined here
"""
self.register_default_tests()
self.test_issues()
self.test_default_issues()
@classmethod
def get_test_cases(cls, filename, file_content):

34
syntribos/tests/fuzz/buffer_overflow.py
View File

@ -32,15 +32,6 @@ class BufferOverflowBody(base_fuzz.BaseFuzzTestCase):
'unknown'
]
def data_driven_failure_cases(self):
if self.failure_keys is None:
return []
failure_assertions = []
for line in self.failure_keys:
failure_assertions.append((self.assertNotIn,
line, self.resp.content))
return failure_assertions
@classmethod
def _get_strings(cls, file_name=None):
return [
@ -50,17 +41,20 @@ class BufferOverflowBody(base_fuzz.BaseFuzzTestCase):
]
def test_case(self):
self.register_default_tests()
self.register_issue(
Issue(test="buffer_overflow",
severity="Medium",
confidence="Low",
text=("A string known to be commonly returned after a "
"successful buffer overflow attack was included "
"in the response. This could indicate a "
"vulnerability to buffer overflow attacks."),
assertions=self.data_driven_failure_cases()))
self.test_issues()
self.test_default_issues()
failed_strings = self.data_driven_failure_cases()
if failed_strings:
self.register_issue(
Issue(test="sql_strings",
severity="Medium",
confidence="Low",
text=("The string(s): \'{0}\', known to be commonly "
"returned after a successful buffer overflow "
"attack, have been found in the response. This "
"could indicate a vulnerability to buffer "
"overflow attacks.").format(failed_strings)
)
)
class BufferOverflowParams(BufferOverflowBody):

38
syntribos/tests/fuzz/sql.py
View File

@ -43,29 +43,25 @@ class SQLInjectionBody(base_fuzz.BaseFuzzTestCase):
'quotation mark',
'syntax',
'ORA-',
'111111']
def data_driven_failure_cases(self):
failure_assertions = []
if self.failure_keys is None:
return []
for line in self.failure_keys:
failure_assertions.append((self.assertNotIn,
line, self.resp.content))
return failure_assertions
'111111'
]
def test_case(self):
self.register_default_tests()
self.register_issue(
Issue(test="sql_strings",
severity="Medium",
confidence="Low",
text=("A string known to be commonly returned after a "
"successful SQL injection attack was included in the "
"response. This could indicate a vulnerability to "
"SQL injection attacks."),
assertions=self.data_driven_failure_cases()))
self.test_issues()
self.test_default_issues()
failed_strings = self.data_driven_failure_cases()
if failed_strings:
self.register_issue(
Issue(test="sql_strings",
severity="Medium",
confidence="Low",
text=("The string(s): \'{0}\', known to be commonly "
"returned after a successful SQL injection attack"
", have been found in the response. This could "
"indicate a vulnerability to SQL injection "
"attacks."
).format(failed_strings)
)
)
class SQLInjectionParams(SQLInjectionBody):

34
syntribos/tests/fuzz/xml_external.py
View File

@ -33,27 +33,21 @@ class XMLExternalEntityBody(base_fuzz.BaseFuzzTestCase):
'disk(0)',
'partition']
def data_driven_failure_cases(self):
failure_assertions = []
if self.failure_keys is None:
return []
for line in self.failure_keys:
failure_assertions.append((self.assertNotIn,
line, self.resp.content))
# return failure_assertions
return [(self.assertTrue, False)]
def test_case(self):
self.register_default_tests()
self.register_issue(
Issue(test="xml_external_entity",
severity="Medium",
text=("A string known to be commonly returned after a "
"successful XML external entity attack was included "
"in the response. This could indicate a "
"vulnerability to XML entity attacks."),
assertions=self.data_driven_failure_cases()))
self.test_issues()
self.test_default_issues()
failed_strings = self.data_driven_failure_cases()
if failed_strings:
self.register_issue(
Issue(test="xml_strings",
severity="Medium",
confidence="Low",
text=("The string(s): \'{0}\', known to be commonly "
"returned after a successful XML external entity "
"attack, have been found in the response. This "
"could indicate a vulnerability to XML external "
"entity attacks.").format(failed_strings)
)
)
class XMLExternalEntityParams(XMLExternalEntityBody):

29
syntribos/tests/fuzz/xss.py
View File

@ -60,23 +60,18 @@ class XSSBody(base_fuzz.BaseFuzzTestCase):
<param name=url value=javascript:alert('XSS')></OBJECT>""",
"""<XML SRC="http://ha.ckers.org/xsstest.xml" ID=I></XML>"""]
def data_driven_failure_cases(self):
failure_assertions = []
if self.failure_keys is None:
return []
for line in self.failure_keys:
failure_assertions.append((self.assertNotIn,
line, self.resp.content))
return failure_assertions
def test_case(self):
if 'html' in self.resp.headers:
self.test_default_issues()
failed_strings = self.data_driven_failure_cases()
if failed_strings and 'html' in self.resp.headers:
self.register_issue(
Issue(test="xss_strings",
severity="High",
text=("A string known to be commonly returned after a "
"successful XSS attack was included "
"in the response. This could indicate a "
"XSS vulnerability"),
assertions=self.data_driven_failure_cases()))
self.test_issues()
severity="Medium",
confidence="Low",
text=("The string(s): \'{0}\', known to be commonly "
"returned after a successful XSS "
"attack, have been found in the response. This "
"could indicate a vulnerability to XSS "
"attacks.").format(failed_strings)
)
)