openstack
/
syntribos
Code Issues Proposed changes

YAML support, basicauth extension, bugfixes 

1) Allows for YAML body in request templates. If a content-type
    is specified for a request template, Syntribos will validate the
    body against the header. This is to prevent templates from silently
    failing to parse and sending unintended data to the target.
 2) Added extension to support basicauth
 3) Lowered confidence ratings for various tests

Change-Id: I672b0e0aa3da1aa6dd7e9a8456da73f0a15759b7
This commit is contained in:
Michael Dong 2018-10-04 13:59:20 -05:00
parent b5cf405d30
commit feb3a59c95
19 changed files with 114 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
syntribos/clients/http/parser.py
View File

@ -25,6 +25,7 @@ from oslo_config import cfg
import six
from six.moves import html_parser
from six.moves.urllib import parse as urlparse
import yaml
from syntribos._i18n import _
@ -63,10 +64,15 @@ class RequestCreator(object):
index = index + 1
method, url, params, version = cls._parse_url_line(lines[0], endpoint)
headers = cls._parse_headers(lines[1:index])
data = cls._parse_data(lines[index + 1:])
content_type = ''
for h in headers:
if h.upper() == 'CONTENT-TYPE':
content_type = headers[h]
break
data, data_type = cls._parse_data(lines[index + 1:], content_type)
return RequestObject(
method=method, url=url, headers=headers, params=params, data=data,
action_field=action_field)
action_field=action_field, data_type=data_type)
@classmethod
def _create_var_obj(cls, var, prefix="", suffix=""):
@ -245,38 +251,63 @@ class RequestCreator(object):
return cls._replace_dict_variables(headers)
@classmethod
def _parse_data(cls, lines):
def _parse_data(cls, lines, content_type=""):
"""Parse the body of the HTTP request (e.g. POST variables)
:param list lines: lines of the HTTP body
:param content_type: Content-type header in template if any
:returns: object representation of body data (JSON or XML)
"""
postdat_regex = r"([\w%]+=[\w%]+&?)+"
data = "\n".join(lines).strip()
data_type = "text"
if not data:
return ""
return '', None
try:
data = json.loads(data)
# TODO(cneill): Make this less hacky
if isinstance(data, list):
data = json.dumps(data)
if isinstance(data, dict):
return cls._replace_dict_variables(data)
return cls._replace_dict_variables(data), 'json'
else:
return cls._replace_str_variables(data)
return cls._replace_str_variables(data), 'str'
except TemplateParseException:
raise
except (TypeError, ValueError):
if 'json' in content_type:
msg = ("The Content-Type header in this template is %s but "
"syntribos cannot parse the request body as json" %
content_type)
raise TemplateParseException(msg)
try:
data = ElementTree.fromstring(data)
data_type = 'xml'
except Exception:
if not re.match(postdat_regex, data):
raise TypeError(_("Template request data does not contain "
"valid JSON or XML data"))
if 'xml' in content_type:
msg = ("The Content-Type header in this template is %s "
"but syntribos cannot parse the request body as xml"
% content_type)
raise TemplateParseException(msg)
try:
data = yaml.safe_load(data)
data_type = 'yaml'
except yaml.YAMLError:
if 'yaml' in content_type:
msg = ("The Content-Type header in this template is %s"
"but syntribos cannot parse the request body as"
"yaml"
% content_type)
raise TemplateParseException(msg)
if not re.match(postdat_regex, data):
raise TypeError(_("Make sure that your request body is"
"valid JSON, XML, or YAML data - be "
"sure to check for typos."))
except Exception:
raise
return data
return data, data_type
@classmethod
def call_external_functions(cls, string):
@ -332,12 +363,7 @@ class RequestCreator(object):
val = func(*args)
except Exception:
msg = _("The reference to the function %s failed to parse "
"correctly, please check the documentation to ensure "
"your function import string adheres to the proper "
"format") % string
raise TemplateParseException(msg)
raise
else:
try:
func_lst = string.split(":")
@ -475,21 +501,23 @@ class RequestHelperMixin(object):
return ele
@staticmethod
def _string_data(data):
def _string_data(data, data_type):
"""Replace various objects types with string representations."""
if isinstance(data, dict):
if data_type == 'json':
return json.dumps(data)
elif isinstance(data, ElementTree.Element):
elif data_type == 'xml':
str_data = ElementTree.tostring(data)
# No way to stop tostring from HTML escaping even if we wanted
h = html_parser.HTMLParser()
return h.unescape(str_data.decode())
elif data_type == 'yaml':
return yaml.dump(data)
else:
return data
@staticmethod
def _replace_iter(string):
"""Fuzz a string."""
"""Replaces action field IDs and meta-variable references."""
if not isinstance(string, six.string_types):
return string
for k, v in list(_iterators.items()):
@ -514,7 +542,7 @@ class RequestHelperMixin(object):
identifier name so that the client only sees example.com/{123} when
it sends the request
"""
return re.sub(r"{[\w]+:", "{", string)
return re.sub(r"(?!{urn:){[\w]+:", "{", string)
def prepare_request(self):
"""Prepare a request for sending off
@ -526,7 +554,7 @@ class RequestHelperMixin(object):
self.data = self._run_iters(self.data, self.action_field)
self.headers = self._run_iters(self.headers, self.action_field)
self.params = self._run_iters(self.params, self.action_field)
self.data = self._string_data(self.data)
self.data = self._string_data(self.data, self.data_type)
self.url = self._run_iters(self.url, self.action_field)
self.url = self._remove_braces(self._remove_attr_names(self.url))
@ -563,7 +591,8 @@ class RequestObject(RequestHelperMixin):
headers=None,
params=None,
data=None,
sanitize=False):
sanitize=False,
data_type=None):
self.method = method
self.url = url
self.action_field = action_field
@ -571,3 +600,4 @@ class RequestObject(RequestHelperMixin):
self.params = params
self.data = data
self.sanitize = sanitize
self.data_type = data_type

7
syntribos/config.py
View File

@ -22,6 +22,7 @@ from syntribos._i18n import _
from syntribos.utils.file_utils import ContentType
from syntribos.utils.file_utils import ExistingDirType
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
OPTS_REGISTERED = False
@ -199,7 +200,7 @@ def list_syntribos_opts():
sample_default="16",
help=_("Maximum number of threads syntribos spawns "
"(experimental)")),
cfg.Opt("templates", type=ContentType("r", 0),
cfg.Opt("templates", type=ContentType("r"),
default="",
sample_default="~/.syntribos/templates",
help=_("A directory of template files, or a single "
@ -222,6 +223,10 @@ def list_syntribos_opts():
"The root directory where the subfolders that make up"
" syntribos' environment (logs, templates, payloads, "
"configuration files, etc.)")),
cfg.StrOpt("meta_vars", sample_default="/path/to/meta.json",
help=_(
"The path to a meta variable definitions file, which "
"will be used when parsing your templates")),
]

0
syntribos/extensions/basic_http/__init__.py Normal file
View File

26
syntribos/extensions/basic_http/client.py Normal file
View File

@ -0,0 +1,26 @@
# Copyright 2018 Rackspace
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import logging
from oslo_config import cfg
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
def basic_auth(user_section='user'):
password = CONF.get(user_section).password or CONF.user.password
username = CONF.get(user_section).username or CONF.user.username
encoded_creds = base64.b64encode("{}:{}".format(username, password))
return "Basic %s" % encoded_creds

2
syntribos/result.py
View File

@ -242,7 +242,7 @@ class IssueTestResult(unittest.TextTestResult):
self.output["errors"] = self.errors
self.output["failures"] = self.failures
formatter_types = {"json": JSONFormatter(self)}
formatter = formatter_types[output_format]
formatter = formatter_types[output_format.lower()]
formatter.report(self.output)
def print_result(self, start_time):

9
syntribos/runner.py
View File

@ -173,8 +173,15 @@ class Runner(object):
:param file_path: the path of the current template
:returns: `dict` of meta variables
"""
path_segments = [""] + os.path.dirname(file_path).split(os.sep)
meta_vars = {}
if CONF.syntribos.meta_vars:
with open(CONF.syntribos.meta_vars, "r") as f:
conf_meta_vars = json.loads(f.read())
for k, v in conf_meta_vars.items():
meta_vars[k] = v
return meta_vars
path_segments = [""] + os.path.dirname(file_path).split(os.sep)
current_path = ""
for seg in path_segments:
current_path = os.path.join(current_path, seg)

2
syntribos/tests/fuzz/base_fuzz.py
View File

@ -38,7 +38,7 @@ class BaseFuzzTestCase(base.BaseTestCase):
payloads = CONF.syntribos.payloads
if not payloads:
payloads = remotes.get(CONF.remote.payloads_uri)
content = ContentType('r', 0)(payloads)
content = ContentType('r')(payloads)
for file_path, _ in content:
if file_path.endswith(".txt"):
file_dir = os.path.split(file_path)[0]

4
syntribos/tests/fuzz/buffer_overflow.py
View File

@ -47,7 +47,7 @@ class BufferOverflowBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="bof_strings",
severity=syntribos.MEDIUM,
confidence=syntribos.LOW,
confidence=syntribos.MEDIUM,
description=("The string(s): '{0}', known to be commonly "
"returned after a successful buffer overflow "
"attack, have been found in the response. This "
@ -59,7 +59,7 @@ class BufferOverflowBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="bof_timing",
severity=syntribos.MEDIUM,
confidence=syntribos.MEDIUM,
confidence=syntribos.LOW,
description=(_("The time it took to resolve a request with a "
"long string was too long compared to the "
"baseline request. This could indicate a "

2
syntribos/tests/fuzz/integer_overflow.py
View File

@ -30,7 +30,7 @@ class IntOverflowBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="int_timing",
severity=syntribos.MEDIUM,
confidence=syntribos.MEDIUM,
confidence=syntribos.LOW,
description=(_("The time it took to resolve a request with an "
"invalid integer was too long compared to the "
"baseline request. This could indicate a "

2
syntribos/tests/fuzz/json_depth_overflow.py
View File

@ -56,7 +56,7 @@ class JSONDepthOverflowBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="json_depth_timing",
severity=syntribos.MEDIUM,
confidence=syntribos.MEDIUM,
confidence=syntribos.LOW,
description=(_("The time it took to resolve a request "
"was too long compared to the "
"baseline request. This could indicate a "

2
syntribos/tests/fuzz/redos.py
View File

@ -30,7 +30,7 @@ class ReDosBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="redos_timing",
severity=syntribos.MEDIUM,
confidence=syntribos.MEDIUM,
confidence=syntribos.LOW,
description=("A response to one of our payload requests has "
"taken too long compared to the baseline "
"request. This could indicate a vulnerability "

2
syntribos/tests/fuzz/sql.py
View File

@ -54,7 +54,7 @@ class SQLInjectionBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="sql_timing",
severity=syntribos.MEDIUM,
confidence=syntribos.MEDIUM,
confidence=syntribos.LOW,
description=(_("A response to one of our payload requests has "
"taken too long compared to the baseline "
"request. This could indicate a vulnerability "

2
syntribos/tests/fuzz/user_defined.py
View File

@ -66,7 +66,7 @@ class UserDefinedVulnBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="user_defined_string_timing",
severity=syntribos.MEDIUM,
confidence=syntribos.MEDIUM,
confidence=syntribos.LOW,
description=(_("A response to one of the payload requests has "
"taken too long compared to the baseline "
"request. This could indicate a vulnerability "

2
syntribos/tests/fuzz/xml_external.py
View File

@ -102,7 +102,7 @@ class XMLExternalEntityBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="xml_timing",
severity=syntribos.MEDIUM,
confidence=syntribos.MEDIUM,
confidence=syntribos.LOW,
description=("The time it took to resolve a request with an "
"invalid URL in the DTD takes too long compared "
"to the baseline request. This could reflect a "

5
syntribos/utils/file_utils.py
View File

@ -47,9 +47,8 @@ class ExistingFileType(ExistingPathType):
class ContentType(ExistingPathType):
"""Reads a file/directory to collect the contents."""
def __init__(self, mode, bufsize):
def __init__(self, mode):
self._mode = mode
self._bufsize = bufsize
self._root = ""
def _fetch_from_dir(self, string):
@ -69,7 +68,7 @@ class ContentType(ExistingPathType):
# Path relative to the "templates" directory specified by user
relative_path = os.path.join(subdir, relative_path)
try:
with open(string, self._mode, self._bufsize) as fp:
with open(string, self._mode) as fp:
return relative_path, fp.read()
except IOError as exc:
self._raise_invalid_file(string, exc=exc)

1
tests/unit/test_datagen.py
View File

@ -224,6 +224,7 @@ class FuzzDatagenUnittest(testtools.TestCase):
"""Test fuzz_request with a JSON-like dict."""
req = post_req(
"/api/v1/{key:val}/path/{otherkey:val2}", data=test_dict)
req.data_type = 'json'
strings = ["test"]
results = [
d for d in fuzz_datagen.fuzz_request(req, strings, "data", "ut")

2
tests/unit/test_file_utils.py
View File

@ -25,7 +25,7 @@ class ConfigUnittest(testtools.TestCase):
ept = utils.ExistingPathType()
edt = utils.ExistingDirType()
eft = utils.ExistingFileType()
tt = utils.ContentType('r', 0)
tt = utils.ContentType('r')
def test_invalid_path_raises_ioerror(self):
"""Test that a random, invalid path raises IOError for each type."""

7
tests/unit/test_http_models.py
View File

@ -81,7 +81,7 @@ class HTTPModelsUnittest(testtools.TestCase):
def test_string_dat_valid_dict(self):
"""Tests RHM._string_data() with a valid dict."""
_dict = {"a": "val", "b": "val2"}
res = rhm._string_data(_dict)
res = rhm._string_data(_dict, 'json')
j_dat = json.loads(res)
self.assertEqual(_dict, j_dat)
@ -96,7 +96,7 @@ class HTTPModelsUnittest(testtools.TestCase):
b = ElementTree.Element("b")
b.text = "hey"
a.append(b)
res = rhm._string_data(a)
res = rhm._string_data(a, 'xml')
self.assertEqual("<a><b>hey</b></a>", res)
def test_string_dat_valid_xml_w_attrs(self):
@ -106,7 +106,7 @@ class HTTPModelsUnittest(testtools.TestCase):
b = ElementTree.Element("b")
b.text = "hey"
a.append(b)
res = rhm._string_data(a)
res = rhm._string_data(a, 'xml')
self.assertEqual('<a key="val"><b>hey</b></a>', res)
def test_run_iters_dict_w_multiple_list(self):
@ -147,6 +147,7 @@ class HTTPModelsUnittest(testtools.TestCase):
def test_prepare_req_action_field_dat(self):
"""Tests RHM.prepare_request() with an ACTION_FIELD var in body."""
r = get_req("/", data={"ACTION_FIELD:var": 1234})
r.data_type = 'json'
prep = r.get_prepared_copy()
j_dat = json.loads(prep.data)
self.assertEqual(1234, j_dat.get("var"))

19
tests/unit/test_http_parser.py
View File

@ -74,7 +74,7 @@ class HTTPParserUnittest(testtools.TestCase):
def test_data_parse_vanilla_json(self):
"""Tests parsing valid JSON data."""
lines = ['{"a": "val", "b": "val2"}']
dat = parser._parse_data(lines)
dat, dat_type = parser._parse_data(lines)
self.assertEqual({"a": "val", "b": "val2"}, dat)
def test_data_parse_invalid_json(self):
@ -88,7 +88,7 @@ class HTTPParserUnittest(testtools.TestCase):
'<?xml version="1.0" encoding="UTF-8"?>',
'<note type="hi"><to>Tove</to><from>Jani</from></note>'
]
dat = parser._parse_data(lines)
dat, dat_type = parser._parse_data(lines)
self.assertEqual("note", dat.tag)
self.assertEqual({"type": "hi"}, dat.attrib)
self.assertEqual("to", dat[0].tag)
@ -98,25 +98,12 @@ class HTTPParserUnittest(testtools.TestCase):
self.assertEqual("Jani", dat[1].text)
self.assertEqual({}, dat[1].attrib)
def test_data_parse_invalid_xml(self):
"""Tests parsing invalid XML data."""
lines = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<note type="hi"><to>Tove<from></to><from>Jani</from></note>'
]
self.assertRaises(TypeError, parser._parse_data, lines)
def test_data_parse_vanilla_postdat(self):
"""Tests parsing valid POST (form) data."""
lines = ["var=val&var2=val2"]
dat = parser._parse_data(lines)
dat, dat_type = parser._parse_data(lines)
self.assertEqual("var=val&var2=val2", dat)
def test_data_parse_invalid_postdat(self):
"""Tests parsing invalid POST (form) data."""
lines = ["var = 1, var2 = 2"]
self.assertRaises(TypeError, parser._parse_data, lines)
def test_call_external_get_uuid(self):
"""Tests calling 'get_uuid' in URL string."""
string = 'GET /v1/CALL_EXTERNAL|'