diff --git a/syntribos/clients/http/parser.py b/syntribos/clients/http/parser.py
index e403f2a0..25e09196 100644
--- a/syntribos/clients/http/parser.py
+++ b/syntribos/clients/http/parser.py
@@ -25,6 +25,7 @@ from oslo_config import cfg
import six
from six.moves import html_parser
from six.moves.urllib import parse as urlparse
+import yaml
from syntribos._i18n import _
@@ -63,10 +64,15 @@ class RequestCreator(object):
index = index + 1
method, url, params, version = cls._parse_url_line(lines[0], endpoint)
headers = cls._parse_headers(lines[1:index])
- data = cls._parse_data(lines[index + 1:])
+ content_type = ''
+ for h in headers:
+ if h.upper() == 'CONTENT-TYPE':
+ content_type = headers[h]
+ break
+ data, data_type = cls._parse_data(lines[index + 1:], content_type)
return RequestObject(
method=method, url=url, headers=headers, params=params, data=data,
- action_field=action_field)
+ action_field=action_field, data_type=data_type)
@classmethod
def _create_var_obj(cls, var, prefix="", suffix=""):
@@ -245,38 +251,63 @@ class RequestCreator(object):
return cls._replace_dict_variables(headers)
@classmethod
- def _parse_data(cls, lines):
+ def _parse_data(cls, lines, content_type=""):
"""Parse the body of the HTTP request (e.g. POST variables)
:param list lines: lines of the HTTP body
+ :param content_type: Content-type header in template if any
:returns: object representation of body data (JSON or XML)
"""
postdat_regex = r"([\w%]+=[\w%]+&?)+"
data = "\n".join(lines).strip()
+ data_type = "text"
if not data:
- return ""
+ return '', None
+
try:
data = json.loads(data)
# TODO(cneill): Make this less hacky
if isinstance(data, list):
data = json.dumps(data)
if isinstance(data, dict):
- return cls._replace_dict_variables(data)
+ return cls._replace_dict_variables(data), 'json'
else:
- return cls._replace_str_variables(data)
+ return cls._replace_str_variables(data), 'str'
except TemplateParseException:
raise
except (TypeError, ValueError):
+ if 'json' in content_type:
+ msg = ("The Content-Type header in this template is %s but "
+ "syntribos cannot parse the request body as json" %
+ content_type)
+ raise TemplateParseException(msg)
try:
data = ElementTree.fromstring(data)
+ data_type = 'xml'
except Exception:
- if not re.match(postdat_regex, data):
- raise TypeError(_("Template request data does not contain "
- "valid JSON or XML data"))
+ if 'xml' in content_type:
+ msg = ("The Content-Type header in this template is %s "
+ "but syntribos cannot parse the request body as xml"
+ % content_type)
+ raise TemplateParseException(msg)
+ try:
+ data = yaml.safe_load(data)
+ data_type = 'yaml'
+ except yaml.YAMLError:
+ if 'yaml' in content_type:
+ msg = ("The Content-Type header in this template is %s"
+ "but syntribos cannot parse the request body as"
+ "yaml"
+ % content_type)
+ raise TemplateParseException(msg)
+ if not re.match(postdat_regex, data):
+ raise TypeError(_("Make sure that your request body is"
+ "valid JSON, XML, or YAML data - be "
+ "sure to check for typos."))
except Exception:
raise
- return data
+ return data, data_type
@classmethod
def call_external_functions(cls, string):
@@ -332,12 +363,7 @@ class RequestCreator(object):
val = func(*args)
except Exception:
- msg = _("The reference to the function %s failed to parse "
- "correctly, please check the documentation to ensure "
- "your function import string adheres to the proper "
- "format") % string
- raise TemplateParseException(msg)
-
+ raise
else:
try:
func_lst = string.split(":")
@@ -475,21 +501,23 @@ class RequestHelperMixin(object):
return ele
@staticmethod
- def _string_data(data):
+ def _string_data(data, data_type):
"""Replace various objects types with string representations."""
- if isinstance(data, dict):
+ if data_type == 'json':
return json.dumps(data)
- elif isinstance(data, ElementTree.Element):
+ elif data_type == 'xml':
str_data = ElementTree.tostring(data)
# No way to stop tostring from HTML escaping even if we wanted
h = html_parser.HTMLParser()
return h.unescape(str_data.decode())
+ elif data_type == 'yaml':
+ return yaml.dump(data)
else:
return data
@staticmethod
def _replace_iter(string):
- """Fuzz a string."""
+ """Replaces action field IDs and meta-variable references."""
if not isinstance(string, six.string_types):
return string
for k, v in list(_iterators.items()):
@@ -514,7 +542,7 @@ class RequestHelperMixin(object):
identifier name so that the client only sees example.com/{123} when
it sends the request
"""
- return re.sub(r"{[\w]+:", "{", string)
+ return re.sub(r"(?!{urn:){[\w]+:", "{", string)
def prepare_request(self):
"""Prepare a request for sending off
@@ -526,7 +554,7 @@ class RequestHelperMixin(object):
self.data = self._run_iters(self.data, self.action_field)
self.headers = self._run_iters(self.headers, self.action_field)
self.params = self._run_iters(self.params, self.action_field)
- self.data = self._string_data(self.data)
+ self.data = self._string_data(self.data, self.data_type)
self.url = self._run_iters(self.url, self.action_field)
self.url = self._remove_braces(self._remove_attr_names(self.url))
@@ -563,7 +591,8 @@ class RequestObject(RequestHelperMixin):
headers=None,
params=None,
data=None,
- sanitize=False):
+ sanitize=False,
+ data_type=None):
self.method = method
self.url = url
self.action_field = action_field
@@ -571,3 +600,4 @@ class RequestObject(RequestHelperMixin):
self.params = params
self.data = data
self.sanitize = sanitize
+ self.data_type = data_type
diff --git a/syntribos/config.py b/syntribos/config.py
index c14d193c..bcfb96f7 100644
--- a/syntribos/config.py
+++ b/syntribos/config.py
@@ -22,6 +22,7 @@ from syntribos._i18n import _
from syntribos.utils.file_utils import ContentType
from syntribos.utils.file_utils import ExistingDirType
+
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
OPTS_REGISTERED = False
@@ -199,7 +200,7 @@ def list_syntribos_opts():
sample_default="16",
help=_("Maximum number of threads syntribos spawns "
"(experimental)")),
- cfg.Opt("templates", type=ContentType("r", 0),
+ cfg.Opt("templates", type=ContentType("r"),
default="",
sample_default="~/.syntribos/templates",
help=_("A directory of template files, or a single "
@@ -222,6 +223,10 @@ def list_syntribos_opts():
"The root directory where the subfolders that make up"
" syntribos' environment (logs, templates, payloads, "
"configuration files, etc.)")),
+ cfg.StrOpt("meta_vars", sample_default="/path/to/meta.json",
+ help=_(
+ "The path to a meta variable definitions file, which "
+ "will be used when parsing your templates")),
]
diff --git a/syntribos/extensions/basic_http/__init__.py b/syntribos/extensions/basic_http/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/syntribos/extensions/basic_http/client.py b/syntribos/extensions/basic_http/client.py
new file mode 100644
index 00000000..55997e3c
--- /dev/null
+++ b/syntribos/extensions/basic_http/client.py
@@ -0,0 +1,26 @@
+# Copyright 2018 Rackspace
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import base64
+import logging
+
+from oslo_config import cfg
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+def basic_auth(user_section='user'):
+ password = CONF.get(user_section).password or CONF.user.password
+ username = CONF.get(user_section).username or CONF.user.username
+ encoded_creds = base64.b64encode("{}:{}".format(username, password))
+ return "Basic %s" % encoded_creds
diff --git a/syntribos/result.py b/syntribos/result.py
index 190ab36d..05321fca 100644
--- a/syntribos/result.py
+++ b/syntribos/result.py
@@ -242,7 +242,7 @@ class IssueTestResult(unittest.TextTestResult):
self.output["errors"] = self.errors
self.output["failures"] = self.failures
formatter_types = {"json": JSONFormatter(self)}
- formatter = formatter_types[output_format]
+ formatter = formatter_types[output_format.lower()]
formatter.report(self.output)
def print_result(self, start_time):
diff --git a/syntribos/runner.py b/syntribos/runner.py
index 22eed2af..8563a73d 100644
--- a/syntribos/runner.py
+++ b/syntribos/runner.py
@@ -173,8 +173,15 @@ class Runner(object):
:param file_path: the path of the current template
:returns: `dict` of meta variables
"""
- path_segments = [""] + os.path.dirname(file_path).split(os.sep)
meta_vars = {}
+ if CONF.syntribos.meta_vars:
+ with open(CONF.syntribos.meta_vars, "r") as f:
+ conf_meta_vars = json.loads(f.read())
+ for k, v in conf_meta_vars.items():
+ meta_vars[k] = v
+ return meta_vars
+
+ path_segments = [""] + os.path.dirname(file_path).split(os.sep)
current_path = ""
for seg in path_segments:
current_path = os.path.join(current_path, seg)
diff --git a/syntribos/tests/fuzz/base_fuzz.py b/syntribos/tests/fuzz/base_fuzz.py
index 0581c1a9..8f4f6ed0 100644
--- a/syntribos/tests/fuzz/base_fuzz.py
+++ b/syntribos/tests/fuzz/base_fuzz.py
@@ -38,7 +38,7 @@ class BaseFuzzTestCase(base.BaseTestCase):
payloads = CONF.syntribos.payloads
if not payloads:
payloads = remotes.get(CONF.remote.payloads_uri)
- content = ContentType('r', 0)(payloads)
+ content = ContentType('r')(payloads)
for file_path, _ in content:
if file_path.endswith(".txt"):
file_dir = os.path.split(file_path)[0]
diff --git a/syntribos/tests/fuzz/buffer_overflow.py b/syntribos/tests/fuzz/buffer_overflow.py
index 6ca05654..d44deb45 100644
--- a/syntribos/tests/fuzz/buffer_overflow.py
+++ b/syntribos/tests/fuzz/buffer_overflow.py
@@ -47,7 +47,7 @@ class BufferOverflowBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="bof_strings",
severity=syntribos.MEDIUM,
- confidence=syntribos.LOW,
+ confidence=syntribos.MEDIUM,
description=("The string(s): '{0}', known to be commonly "
"returned after a successful buffer overflow "
"attack, have been found in the response. This "
@@ -59,7 +59,7 @@ class BufferOverflowBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="bof_timing",
severity=syntribos.MEDIUM,
- confidence=syntribos.MEDIUM,
+ confidence=syntribos.LOW,
description=(_("The time it took to resolve a request with a "
"long string was too long compared to the "
"baseline request. This could indicate a "
diff --git a/syntribos/tests/fuzz/integer_overflow.py b/syntribos/tests/fuzz/integer_overflow.py
index 18fcb322..d3df657e 100644
--- a/syntribos/tests/fuzz/integer_overflow.py
+++ b/syntribos/tests/fuzz/integer_overflow.py
@@ -30,7 +30,7 @@ class IntOverflowBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="int_timing",
severity=syntribos.MEDIUM,
- confidence=syntribos.MEDIUM,
+ confidence=syntribos.LOW,
description=(_("The time it took to resolve a request with an "
"invalid integer was too long compared to the "
"baseline request. This could indicate a "
diff --git a/syntribos/tests/fuzz/json_depth_overflow.py b/syntribos/tests/fuzz/json_depth_overflow.py
index e57661fb..2e7f168f 100644
--- a/syntribos/tests/fuzz/json_depth_overflow.py
+++ b/syntribos/tests/fuzz/json_depth_overflow.py
@@ -56,7 +56,7 @@ class JSONDepthOverflowBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="json_depth_timing",
severity=syntribos.MEDIUM,
- confidence=syntribos.MEDIUM,
+ confidence=syntribos.LOW,
description=(_("The time it took to resolve a request "
"was too long compared to the "
"baseline request. This could indicate a "
diff --git a/syntribos/tests/fuzz/redos.py b/syntribos/tests/fuzz/redos.py
index cf4536ce..9c983c19 100644
--- a/syntribos/tests/fuzz/redos.py
+++ b/syntribos/tests/fuzz/redos.py
@@ -30,7 +30,7 @@ class ReDosBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="redos_timing",
severity=syntribos.MEDIUM,
- confidence=syntribos.MEDIUM,
+ confidence=syntribos.LOW,
description=("A response to one of our payload requests has "
"taken too long compared to the baseline "
"request. This could indicate a vulnerability "
diff --git a/syntribos/tests/fuzz/sql.py b/syntribos/tests/fuzz/sql.py
index 0765c762..12f475b9 100644
--- a/syntribos/tests/fuzz/sql.py
+++ b/syntribos/tests/fuzz/sql.py
@@ -54,7 +54,7 @@ class SQLInjectionBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="sql_timing",
severity=syntribos.MEDIUM,
- confidence=syntribos.MEDIUM,
+ confidence=syntribos.LOW,
description=(_("A response to one of our payload requests has "
"taken too long compared to the baseline "
"request. This could indicate a vulnerability "
diff --git a/syntribos/tests/fuzz/user_defined.py b/syntribos/tests/fuzz/user_defined.py
index ed5d43dd..1b71fb7a 100644
--- a/syntribos/tests/fuzz/user_defined.py
+++ b/syntribos/tests/fuzz/user_defined.py
@@ -66,7 +66,7 @@ class UserDefinedVulnBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="user_defined_string_timing",
severity=syntribos.MEDIUM,
- confidence=syntribos.MEDIUM,
+ confidence=syntribos.LOW,
description=(_("A response to one of the payload requests has "
"taken too long compared to the baseline "
"request. This could indicate a vulnerability "
diff --git a/syntribos/tests/fuzz/xml_external.py b/syntribos/tests/fuzz/xml_external.py
index 42943a63..8786a17e 100644
--- a/syntribos/tests/fuzz/xml_external.py
+++ b/syntribos/tests/fuzz/xml_external.py
@@ -102,7 +102,7 @@ class XMLExternalEntityBody(base_fuzz.BaseFuzzTestCase):
self.register_issue(
defect_type="xml_timing",
severity=syntribos.MEDIUM,
- confidence=syntribos.MEDIUM,
+ confidence=syntribos.LOW,
description=("The time it took to resolve a request with an "
"invalid URL in the DTD takes too long compared "
"to the baseline request. This could reflect a "
diff --git a/syntribos/utils/file_utils.py b/syntribos/utils/file_utils.py
index 21b0547f..da5c22c4 100644
--- a/syntribos/utils/file_utils.py
+++ b/syntribos/utils/file_utils.py
@@ -47,9 +47,8 @@ class ExistingFileType(ExistingPathType):
class ContentType(ExistingPathType):
"""Reads a file/directory to collect the contents."""
- def __init__(self, mode, bufsize):
+ def __init__(self, mode):
self._mode = mode
- self._bufsize = bufsize
self._root = ""
def _fetch_from_dir(self, string):
@@ -69,7 +68,7 @@ class ContentType(ExistingPathType):
# Path relative to the "templates" directory specified by user
relative_path = os.path.join(subdir, relative_path)
try:
- with open(string, self._mode, self._bufsize) as fp:
+ with open(string, self._mode) as fp:
return relative_path, fp.read()
except IOError as exc:
self._raise_invalid_file(string, exc=exc)
diff --git a/tests/unit/test_datagen.py b/tests/unit/test_datagen.py
index bbc6999d..3a57c8e9 100644
--- a/tests/unit/test_datagen.py
+++ b/tests/unit/test_datagen.py
@@ -224,6 +224,7 @@ class FuzzDatagenUnittest(testtools.TestCase):
"""Test fuzz_request with a JSON-like dict."""
req = post_req(
"/api/v1/{key:val}/path/{otherkey:val2}", data=test_dict)
+ req.data_type = 'json'
strings = ["test"]
results = [
d for d in fuzz_datagen.fuzz_request(req, strings, "data", "ut")
diff --git a/tests/unit/test_file_utils.py b/tests/unit/test_file_utils.py
index e63e5c9e..25f1bf40 100644
--- a/tests/unit/test_file_utils.py
+++ b/tests/unit/test_file_utils.py
@@ -25,7 +25,7 @@ class ConfigUnittest(testtools.TestCase):
ept = utils.ExistingPathType()
edt = utils.ExistingDirType()
eft = utils.ExistingFileType()
- tt = utils.ContentType('r', 0)
+ tt = utils.ContentType('r')
def test_invalid_path_raises_ioerror(self):
"""Test that a random, invalid path raises IOError for each type."""
diff --git a/tests/unit/test_http_models.py b/tests/unit/test_http_models.py
index 36da314a..4fdcc4ea 100644
--- a/tests/unit/test_http_models.py
+++ b/tests/unit/test_http_models.py
@@ -81,7 +81,7 @@ class HTTPModelsUnittest(testtools.TestCase):
def test_string_dat_valid_dict(self):
"""Tests RHM._string_data() with a valid dict."""
_dict = {"a": "val", "b": "val2"}
- res = rhm._string_data(_dict)
+ res = rhm._string_data(_dict, 'json')
j_dat = json.loads(res)
self.assertEqual(_dict, j_dat)
@@ -96,7 +96,7 @@ class HTTPModelsUnittest(testtools.TestCase):
b = ElementTree.Element("b")
b.text = "hey"
a.append(b)
- res = rhm._string_data(a)
+ res = rhm._string_data(a, 'xml')
self.assertEqual("hey", res)
def test_string_dat_valid_xml_w_attrs(self):
@@ -106,7 +106,7 @@ class HTTPModelsUnittest(testtools.TestCase):
b = ElementTree.Element("b")
b.text = "hey"
a.append(b)
- res = rhm._string_data(a)
+ res = rhm._string_data(a, 'xml')
self.assertEqual('hey', res)
def test_run_iters_dict_w_multiple_list(self):
@@ -147,6 +147,7 @@ class HTTPModelsUnittest(testtools.TestCase):
def test_prepare_req_action_field_dat(self):
"""Tests RHM.prepare_request() with an ACTION_FIELD var in body."""
r = get_req("/", data={"ACTION_FIELD:var": 1234})
+ r.data_type = 'json'
prep = r.get_prepared_copy()
j_dat = json.loads(prep.data)
self.assertEqual(1234, j_dat.get("var"))
diff --git a/tests/unit/test_http_parser.py b/tests/unit/test_http_parser.py
index eca4ef0c..80b1a9bc 100644
--- a/tests/unit/test_http_parser.py
+++ b/tests/unit/test_http_parser.py
@@ -74,7 +74,7 @@ class HTTPParserUnittest(testtools.TestCase):
def test_data_parse_vanilla_json(self):
"""Tests parsing valid JSON data."""
lines = ['{"a": "val", "b": "val2"}']
- dat = parser._parse_data(lines)
+ dat, dat_type = parser._parse_data(lines)
self.assertEqual({"a": "val", "b": "val2"}, dat)
def test_data_parse_invalid_json(self):
@@ -88,7 +88,7 @@ class HTTPParserUnittest(testtools.TestCase):
'',
'ToveJani'
]
- dat = parser._parse_data(lines)
+ dat, dat_type = parser._parse_data(lines)
self.assertEqual("note", dat.tag)
self.assertEqual({"type": "hi"}, dat.attrib)
self.assertEqual("to", dat[0].tag)
@@ -98,25 +98,12 @@ class HTTPParserUnittest(testtools.TestCase):
self.assertEqual("Jani", dat[1].text)
self.assertEqual({}, dat[1].attrib)
- def test_data_parse_invalid_xml(self):
- """Tests parsing invalid XML data."""
- lines = [
- '',
- 'ToveJani'
- ]
- self.assertRaises(TypeError, parser._parse_data, lines)
-
def test_data_parse_vanilla_postdat(self):
"""Tests parsing valid POST (form) data."""
lines = ["var=val&var2=val2"]
- dat = parser._parse_data(lines)
+ dat, dat_type = parser._parse_data(lines)
self.assertEqual("var=val&var2=val2", dat)
- def test_data_parse_invalid_postdat(self):
- """Tests parsing invalid POST (form) data."""
- lines = ["var = 1, var2 = 2"]
- self.assertRaises(TypeError, parser._parse_data, lines)
-
def test_call_external_get_uuid(self):
"""Tests calling 'get_uuid' in URL string."""
string = 'GET /v1/CALL_EXTERNAL|'