summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMichael Dong <michael.dong@rackspace.com>2017-08-16 20:12:39 -0500
committerMichael Dong <michael.dong@rackspace.com>2017-09-18 13:23:00 -0500
commitc4586a374b2e02942b89bf1e9248ae00d7cd2d55 (patch)
tree2cb4457ec66b80036491c1c9609a8106e8630f40
parentcb458c03db79507e55b1d55220cecd123090c4dc (diff)
Improve performance by multithreading test calls0.4.0
This change: 1) rewrites the runner to spawn a thread pool for each template and assigns a worker for each test case 2) makes the output colorized by default 3) makes minor changes to the output Change-Id: I49906f5daaa339ca9429913680203c762a0ad9fe
Notes
Notes (review): Code-Review+2: Charles Neill <charles.neill@rackspace.com> Workflow+1: Michael Dong <michael.dong@rackspace.com> Verified+2: Jenkins Submitted-by: Jenkins Submitted-at: Fri, 13 Oct 2017 18:32:18 +0000 Reviewed-on: https://review.openstack.org/496953 Project: openstack/syntribos Branch: refs/heads/master
-rw-r--r--syntribos/clients/http/base_http_client.py3
-rw-r--r--syntribos/clients/http/client.py2
-rw-r--r--syntribos/clients/http/debug_logger.py43
-rw-r--r--syntribos/config.py7
-rw-r--r--syntribos/result.py44
-rw-r--r--syntribos/runner.py86
-rw-r--r--syntribos/tests/base.py7
-rw-r--r--syntribos/tests/fuzz/base_fuzz.py10
-rw-r--r--syntribos/tests/fuzz/buffer_overflow.py1
-rw-r--r--syntribos/utils/cli.py29
-rw-r--r--syntribos/utils/config_fixture.py2
-rw-r--r--tests/unit/test_ascii_colors.py4
-rw-r--r--tests/unit/test_results.py2
13 files changed, 129 insertions, 111 deletions
diff --git a/syntribos/clients/http/base_http_client.py b/syntribos/clients/http/base_http_client.py
index 0702398..ce09fc9 100644
--- a/syntribos/clients/http/base_http_client.py
+++ b/syntribos/clients/http/base_http_client.py
@@ -80,4 +80,5 @@ class HTTPClient(object):
80 'data': data}, **requestslib_kwargs) 80 'data': data}, **requestslib_kwargs)
81 81
82 # Make the request 82 # Make the request
83 return requests.request(method, url, **requestslib_kwargs) 83 return requests.request(method, url, allow_redirects=False,
84 **requestslib_kwargs)
diff --git a/syntribos/clients/http/client.py b/syntribos/clients/http/client.py
index 63bb80a..211833e 100644
--- a/syntribos/clients/http/client.py
+++ b/syntribos/clients/http/client.py
@@ -53,7 +53,7 @@ class SynHTTPClient(HTTPClient):
53 return (response, signals) 53 return (response, signals)
54 54
55 def send_request(self, request_obj): 55 def send_request(self, request_obj):
56 """This sends a request based on a RequestOjbect. 56 """This sends a request based on a RequestObject.
57 57
58 RequestObjects are generated by a parser (e.g. 58 RequestObjects are generated by a parser (e.g.
59 :class:`syntribos.clients.http.parser.RequestCreator`) from request 59 :class:`syntribos.clients.http.parser.RequestCreator`) from request
diff --git a/syntribos/clients/http/debug_logger.py b/syntribos/clients/http/debug_logger.py
index b962b1d..4d90c37 100644
--- a/syntribos/clients/http/debug_logger.py
+++ b/syntribos/clients/http/debug_logger.py
@@ -17,6 +17,7 @@
17# limitations under the License. 17# limitations under the License.
18from copy import deepcopy 18from copy import deepcopy
19import logging 19import logging
20import threading
20from time import time 21from time import time
21 22
22import requests 23import requests
@@ -27,6 +28,8 @@ import syntribos.checks.http as http_checks
27import syntribos.signal 28import syntribos.signal
28from syntribos.utils import string_utils 29from syntribos.utils import string_utils
29 30
31lock = threading.Lock()
32
30 33
31def log_http_transaction(log, level=logging.DEBUG): 34def log_http_transaction(log, level=logging.DEBUG):
32 """Decorator used for logging requests/response in clients. 35 """Decorator used for logging requests/response in clients.
@@ -59,20 +62,13 @@ def log_http_transaction(log, level=logging.DEBUG):
59 sent to the request() method, to the provided log at the provided 62 sent to the request() method, to the provided log at the provided
60 log level. 63 log level.
61 """ 64 """
65
62 kwargs_copy = deepcopy(kwargs) 66 kwargs_copy = deepcopy(kwargs)
63 if kwargs_copy.get("sanitize"): 67 if kwargs_copy.get("sanitize"):
64 kwargs_copy = string_utils.sanitize_secrets(kwargs_copy) 68 kwargs_copy = string_utils.sanitize_secrets(kwargs_copy)
65 logline = '{0} {1}'.format(args, string_utils.compress( 69 logline_obj = '{0} {1}'.format(args, string_utils.compress(
66 kwargs_copy)) 70 kwargs_copy))
67 71
68 try:
69 log.debug(_safe_decode(logline))
70 except Exception as exception:
71 # Ignore all exceptions that happen in logging, then log them
72 log.info('Exception occurred while logging signature of '
73 'calling method in http client')
74 log.exception(exception)
75
76 # Make the request and time its execution 72 # Make the request and time its execution
77 response = None 73 response = None
78 no_resp_time = None 74 no_resp_time = None
@@ -132,7 +128,7 @@ def log_http_transaction(log, level=logging.DEBUG):
132 request_headers = string_utils.sanitize_secrets( 128 request_headers = string_utils.sanitize_secrets(
133 request_headers) 129 request_headers)
134 request_body = string_utils.sanitize_secrets(request_body) 130 request_body = string_utils.sanitize_secrets(request_body)
135 logline = ''.join([ 131 logline_req = ''.join([
136 '\n{0}\nREQUEST SENT\n{0}\n'.format('-' * 12), 132 '\n{0}\nREQUEST SENT\n{0}\n'.format('-' * 12),
137 'request method.......: {0}\n'.format(response.request.method), 133 'request method.......: {0}\n'.format(response.request.method),
138 'request url..........: {0}\n'.format(string_utils.compress( 134 'request url..........: {0}\n'.format(string_utils.compress(
@@ -145,15 +141,7 @@ def log_http_transaction(log, level=logging.DEBUG):
145 'request body size....: {0}\n'.format(req_body_len), 141 'request body size....: {0}\n'.format(req_body_len),
146 'request body.........: {0}\n'.format(string_utils.compress 142 'request body.........: {0}\n'.format(string_utils.compress
147 (request_body))]) 143 (request_body))])
148 144 logline_rsp = ''.join([
149 try:
150 log.log(level, _safe_decode(logline))
151 except Exception as exception:
152 # Ignore all exceptions that happen in logging, then log them
153 log.log(level, '\n{0}\nREQUEST INFO\n{0}\n'.format('-' * 12))
154 log.exception(exception)
155
156 logline = ''.join([
157 '\n{0}\nRESPONSE RECEIVED\n{0}\n'.format('-' * 17), 145 '\n{0}\nRESPONSE RECEIVED\n{0}\n'.format('-' * 17),
158 'response status..: {0}\n'.format(response), 146 'response status..: {0}\n'.format(response),
159 'response headers.: {0}\n'.format(response.headers), 147 'response headers.: {0}\n'.format(response.headers),
@@ -162,12 +150,27 @@ def log_http_transaction(log, level=logging.DEBUG):
162 'response size....: {0}\n'.format(len(response.content)), 150 'response size....: {0}\n'.format(len(response.content)),
163 'response body....: {0}\n'.format(response_content), 151 'response body....: {0}\n'.format(response_content),
164 '-' * 79]) 152 '-' * 79])
153 lock.acquire()
154 try:
155 log.log(level, _safe_decode(logline_req))
156 except Exception as exception:
157 # Ignore all exceptions that happen in logging, then log them
158 log.log(level, '\n{0}\nREQUEST INFO\n{0}\n'.format('-' * 12))
159 log.exception(exception)
165 try: 160 try:
166 log.log(level, _safe_decode(logline)) 161 log.log(level, _safe_decode(logline_rsp))
167 except Exception as exception: 162 except Exception as exception:
168 # Ignore all exceptions that happen in logging, then log them 163 # Ignore all exceptions that happen in logging, then log them
169 log.log(level, '\n{0}\nRESPONSE INFO\n{0}\n'.format('-' * 13)) 164 log.log(level, '\n{0}\nRESPONSE INFO\n{0}\n'.format('-' * 13))
170 log.exception(exception) 165 log.exception(exception)
166 try:
167 log.debug(_safe_decode(logline_obj))
168 except Exception as exception:
169 # Ignore all exceptions that happen in logging, then log them
170 log.info('Exception occurred while logging signature of '
171 'calling method in http client')
172 log.exception(exception)
173 lock.release()
171 return (response, signals) 174 return (response, signals)
172 return _wrapper 175 return _wrapper
173 return _decorator 176 return _decorator
diff --git a/syntribos/config.py b/syntribos/config.py
index 8072537..c14d193 100644
--- a/syntribos/config.py
+++ b/syntribos/config.py
@@ -157,7 +157,8 @@ def list_cli_opts():
157 default=[""], sample_default=["SQL", "XSS"], 157 default=[""], sample_default=["SQL", "XSS"],
158 help=_("Test types to be excluded from " 158 help=_("Test types to be excluded from "
159 "current run against the target API")), 159 "current run against the target API")),
160 cfg.BoolOpt("colorize", dest="colorize", short="cl", default=False, 160 cfg.BoolOpt("no_colorize", dest="no_colorize", short="ncl",
161 default=False,
161 help=_("Enable color in syntribos terminal output")), 162 help=_("Enable color in syntribos terminal output")),
162 cfg.StrOpt("outfile", short="o", 163 cfg.StrOpt("outfile", short="o",
163 sample_default="out.json", help=_("File to print " 164 sample_default="out.json", help=_("File to print "
@@ -194,6 +195,10 @@ def list_syntribos_opts():
194 cfg.StrOpt("endpoint", default="", 195 cfg.StrOpt("endpoint", default="",
195 sample_default="http://localhost/app", 196 sample_default="http://localhost/app",
196 help=_("The target host to be tested")), 197 help=_("The target host to be tested")),
198 cfg.IntOpt("threads", default=16,
199 sample_default="16",
200 help=_("Maximum number of threads syntribos spawns "
201 "(experimental)")),
197 cfg.Opt("templates", type=ContentType("r", 0), 202 cfg.Opt("templates", type=ContentType("r", 0),
198 default="", 203 default="",
199 sample_default="~/.syntribos/templates", 204 sample_default="~/.syntribos/templates",
diff --git a/syntribos/result.py b/syntribos/result.py
index 0f74686..190ab36 100644
--- a/syntribos/result.py
+++ b/syntribos/result.py
@@ -11,6 +11,7 @@
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14import threading
14import time 15import time
15import unittest 16import unittest
16 17
@@ -23,6 +24,7 @@ from syntribos.runner import Runner
23import syntribos.utils.remotes 24import syntribos.utils.remotes
24 25
25CONF = cfg.CONF 26CONF = cfg.CONF
27lock = threading.Lock()
26 28
27 29
28class IssueTestResult(unittest.TextTestResult): 30class IssueTestResult(unittest.TextTestResult):
@@ -38,7 +40,7 @@ class IssueTestResult(unittest.TextTestResult):
38 "MEDIUM": 0, 40 "MEDIUM": 0,
39 "HIGH": 0 41 "HIGH": 0
40 } 42 }
41 stats = {"errors": 0, "failures": 0, "successes": 0} 43 stats = {"errors": 0, "unique_failures": 0, "successes": 0}
42 severity_counter_dict = {} 44 severity_counter_dict = {}
43 testsRunSinceLastPrint = 0 45 testsRunSinceLastPrint = 0
44 failure_id = 0 46 failure_id = 0
@@ -84,6 +86,7 @@ class IssueTestResult(unittest.TextTestResult):
84 :type test: :class:`syntribos.tests.base.BaseTestCase` 86 :type test: :class:`syntribos.tests.base.BaseTestCase`
85 :param tuple err: Tuple of format ``(type, value, traceback)`` 87 :param tuple err: Tuple of format ``(type, value, traceback)``
86 """ 88 """
89 lock.acquire()
87 for issue in test.failures: 90 for issue in test.failures:
88 defect_type = issue.defect_type 91 defect_type = issue.defect_type
89 if any([ 92 if any([
@@ -174,7 +177,7 @@ class IssueTestResult(unittest.TextTestResult):
174 "signals": signals 177 "signals": signals
175 } 178 }
176 failure_obj["instances"].append(instance_obj) 179 failure_obj["instances"].append(instance_obj)
177 self.stats["failures"] += 1 180 self.stats["unique_failures"] += 1
178 self.output["stats"]["severity"][sev_rating] += 1 181 self.output["stats"]["severity"][sev_rating] += 1
179 else: 182 else:
180 instance_obj = None 183 instance_obj = None
@@ -196,8 +199,9 @@ class IssueTestResult(unittest.TextTestResult):
196 "signals": signals 199 "signals": signals
197 } 200 }
198 failure_obj["instances"].append(instance_obj) 201 failure_obj["instances"].append(instance_obj)
199 self.stats["failures"] += 1 202 self.stats["unique_failures"] += 1
200 self.output["stats"]["severity"][sev_rating] += 1 203 self.output["stats"]["severity"][sev_rating] += 1
204 lock.release()
201 205
202 def addError(self, test, err): 206 def addError(self, test, err):
203 """Duplicates parent class addError functionality. 207 """Duplicates parent class addError functionality.
@@ -207,11 +211,19 @@ class IssueTestResult(unittest.TextTestResult):
207 :param err: 211 :param err:
208 :type tuple: Tuple of format ``(type, value, traceback)`` 212 :type tuple: Tuple of format ``(type, value, traceback)``
209 """ 213 """
210 self.errors.append({ 214 with lock:
211 "test": self.getDescription(test), 215 for e in self.errors:
212 "error": self._exc_info_to_string(err, test) 216 if e['error'] == self._exc_info_to_string(err, test):
213 }) 217 if self.getDescription(test) in e['test']:
214 self.stats["errors"] += 1 218 return
219 e['test'].append(self.getDescription(test))
220 self.stats["errors"] += 1
221 return
222 self.errors.append({
223 "test": [self.getDescription(test)],
224 "error": self._exc_info_to_string(err, test)
225 })
226 self.stats["errors"] += 1
215 227
216 def addSuccess(self, test): 228 def addSuccess(self, test):
217 """Duplicates parent class addSuccess functionality. 229 """Duplicates parent class addSuccess functionality.
@@ -219,7 +231,8 @@ class IssueTestResult(unittest.TextTestResult):
219 :param test: The test that was run 231 :param test: The test that was run
220 :type test: :class:`syntribos.tests.base.BaseTestCase` 232 :type test: :class:`syntribos.tests.base.BaseTestCase`
221 """ 233 """
222 self.stats["successes"] += 1 234 with lock:
235 self.stats["successes"] += 1
223 236
224 def printErrors(self, output_format): 237 def printErrors(self, output_format):
225 """Print out each :class:`syntribos.issue.Issue` that was encountered 238 """Print out each :class:`syntribos.issue.Issue` that was encountered
@@ -241,18 +254,19 @@ class IssueTestResult(unittest.TextTestResult):
241 """Print the path to the log folder for this run.""" 254 """Print the path to the log folder for this run."""
242 test_log = Runner.log_path 255 test_log = Runner.log_path
243 run_time = time.time() - start_time 256 run_time = time.time() - start_time
244 num_fail = self.stats["failures"] 257 num_fail = self.stats["unique_failures"]
245 num_err = self.stats["errors"] 258 num_err = self.stats["errors"]
246 print("\n{sep}\nTotal: Ran {num} test{suff} in {time:.3f}s".format( 259 print("\n{sep}\nTotal: Ran {num} test{suff} in {time:.3f}s".format(
247 sep=syntribos.SEP, 260 sep=syntribos.SEP,
248 num=self.testsRun, 261 num=self.testsRun,
249 suff="s" * bool(self.testsRun - 1), 262 suff="s" * bool(self.testsRun - 1),
250 time=run_time)) 263 time=run_time))
251 print("Total: {f} failure{fsuff} and {e} error{esuff}".format( 264 print("Total: {f} unique failure{fsuff} "
252 f=num_fail, 265 "and {e} unique error{esuff}".format(
253 e=num_err, 266 f=num_fail,
254 fsuff="s" * bool(num_fail - 1), 267 e=num_err,
255 esuff="s" * bool(num_err - 1))) 268 fsuff="s" * bool(num_fail - 1),
269 esuff="s" * bool(num_err - 1)))
256 if test_log: 270 if test_log:
257 print(syntribos.SEP) 271 print(syntribos.SEP)
258 print(_("LOG PATH...: %s") % test_log) 272 print(_("LOG PATH...: %s") % test_log)
diff --git a/syntribos/runner.py b/syntribos/runner.py
index 4b01406..22eed2a 100644
--- a/syntribos/runner.py
+++ b/syntribos/runner.py
@@ -13,9 +13,11 @@
13# limitations under the License. 13# limitations under the License.
14import json 14import json
15import logging 15import logging
16from multiprocessing.dummy import Pool as ThreadPool
16import os 17import os
17import pkgutil 18import pkgutil
18import sys 19import sys
20import threading
19import time 21import time
20import traceback 22import traceback
21import unittest 23import unittest
@@ -39,6 +41,7 @@ result = None
39user_base_dir = None 41user_base_dir = None
40CONF = cfg.CONF 42CONF = cfg.CONF
41LOG = logging.getLogger(__name__) 43LOG = logging.getLogger(__name__)
44lock = threading.Lock()
42 45
43 46
44class Runner(object): 47class Runner(object):
@@ -125,6 +128,7 @@ class Runner(object):
125 LOG = logging.getLogger() 128 LOG = logging.getLogger()
126 LOG.handlers = [log_handle] 129 LOG.handlers = [log_handle]
127 LOG.setLevel(logging.DEBUG) 130 LOG.setLevel(logging.DEBUG)
131 logging.getLogger("urllib3").setLevel(logging.WARNING)
128 return LOG 132 return LOG
129 133
130 @classmethod 134 @classmethod
@@ -258,7 +262,6 @@ class Runner(object):
258 cls.meta_dir_dict[meta_path] = json.loads(file_content) 262 cls.meta_dir_dict[meta_path] = json.loads(file_content)
259 except Exception: 263 except Exception:
260 print("Unable to parse %s, skipping..." % file_path) 264 print("Unable to parse %s, skipping..." % file_path)
261
262 for file_path, req_str in templates_dir: 265 for file_path, req_str in templates_dir:
263 if "meta.json" in file_path: 266 if "meta.json" in file_path:
264 continue 267 continue
@@ -331,7 +334,7 @@ class Runner(object):
331 if len(test_cases) > 0: 334 if len(test_cases) > 0:
332 for test in test_cases: 335 for test in test_cases:
333 if test: 336 if test:
334 cls.run_test(test, result) 337 cls.run_test(test)
335 338
336 @classmethod 339 @classmethod
337 def dry_run_report(cls, output): 340 def dry_run_report(cls, output):
@@ -359,6 +362,7 @@ class Runner(object):
359 362
360 :return: None 363 :return: None
361 """ 364 """
365 pool = ThreadPool(CONF.syntribos.threads)
362 try: 366 try:
363 template_start_time = time.time() 367 template_start_time = time.time()
364 failures = 0 368 failures = 0
@@ -367,19 +371,18 @@ class Runner(object):
367 for test_name, test_class in list_of_tests: 371 for test_name, test_class in list_of_tests:
368 test_class.test_id = cls.current_test_id 372 test_class.test_id = cls.current_test_id
369 cls.current_test_id += 5 373 cls.current_test_id += 5
370 log_string = "[{test_id}] : {name}".format( 374
371 test_id=test_class.test_id, name=test_name)
372 result_string = "[{test_id}] : {name}".format( 375 result_string = "[{test_id}] : {name}".format(
373 test_id=cli.colorize( 376 test_id=cli.colorize(
374 test_class.test_id, color="green"), 377 test_class.test_id, color="green"),
375 name=test_name.replace("_", " ").capitalize()) 378 name=test_name.replace("_", " ").capitalize())
376 if not CONF.colorize: 379 if CONF.no_colorize:
377 result_string = result_string.ljust(55) 380 result_string = result_string.ljust(55)
378 else: 381 else:
379 result_string = result_string.ljust(60) 382 result_string = result_string.ljust(60)
380 LOG.debug(log_string)
381 try: 383 try:
382 test_class.send_init_request(file_path, req_str, meta_vars) 384 test_class.create_init_request(file_path, req_str,
385 meta_vars)
383 except Exception: 386 except Exception:
384 print(_( 387 print(_(
385 "Error in parsing template:\n %s\n" 388 "Error in parsing template:\n %s\n"
@@ -388,40 +391,33 @@ class Runner(object):
388 break 391 break
389 test_cases = list( 392 test_cases = list(
390 test_class.get_test_cases(file_path, req_str)) 393 test_class.get_test_cases(file_path, req_str))
391 if len(test_cases) > 0: 394 total_tests = len(test_cases)
395 if total_tests > 0:
396 log_string = "[{test_id}] : {name}".format(
397 test_id=test_class.test_id, name=test_name)
398 LOG.debug(log_string)
399 last_failures = result.stats['unique_failures']
400 last_errors = result.stats['errors']
392 p_bar = cli.ProgressBar( 401 p_bar = cli.ProgressBar(
393 message=result_string, total_len=len(test_cases)) 402 message=result_string, total_len=total_tests)
394 last_failures = result.stats["failures"] 403 test_class.send_init_request(file_path, req_str, meta_vars)
395 last_errors = result.stats["errors"] 404
396 for test in test_cases: 405 # This line runs the tests
397 if test: 406 pool.map(lambda t: cls.run_test(t, p_bar), test_cases)
398 cls.run_test(test, result) 407
399 p_bar.increment(1) 408 failures = result.stats['unique_failures'] - last_failures
400 p_bar.print_bar() 409 errors = result.stats['errors'] - last_errors
401 failures = result.stats["failures"] - last_failures 410 failures_str = cli.colorize_by_percent(
402 errors = result.stats["errors"] - last_errors 411 failures, total_tests, "red")
403 total_tests = len(test_cases) 412
404 if failures > total_tests * 0.90:
405 # More than 90 percent failure
406 failures = cli.colorize(failures, "red")
407 elif failures > total_tests * 0.45:
408 # More than 45 percent failure
409 failures = cli.colorize(failures, "yellow")
410 elif failures > total_tests * 0.15:
411 # More than 15 percent failure
412 failures = cli.colorize(failures, "blue")
413 if errors: 413 if errors:
414 last_failures = result.stats["failures"] 414 errors_str = cli.colorize(errors, "red")
415 last_errors = result.stats["errors"]
416 errors = cli.colorize(errors, "red")
417 print(_( 415 print(_(
418 " : %(fail)s Failure(s), %(err)s Error(s)\r") % { 416 " : %(fail)s Failure(s), %(err)s Error(s)\r") % {
419 "fail": failures, "err": errors}) 417 "fail": failures_str, "err": errors_str})
420 else: 418 else:
421 last_failures = result.stats["failures"] 419 print(_(
422 print( 420 " : %s Failure(s), 0 Error(s)\r") % failures_str)
423 _(
424 " : %s Failure(s), 0 Error(s)\r") % failures)
425 421
426 run_time = time.time() - template_start_time 422 run_time = time.time() - template_start_time
427 LOG.info(_("Run time: %s sec."), run_time) 423 LOG.info(_("Run time: %s sec."), run_time)
@@ -440,26 +436,34 @@ class Runner(object):
440 result.print_result(cls.start_time) 436 result.print_result(cls.start_time)
441 cleanup.delete_temps() 437 cleanup.delete_temps()
442 print(_("Exiting...")) 438 print(_("Exiting..."))
439 pool.close()
440 pool.join()
443 exit(0) 441 exit(0)
444 print(_('Resuming...')) 442 print(_('Resuming...'))
445 except KeyboardInterrupt: 443 except KeyboardInterrupt:
446 result.print_result(cls.start_time) 444 result.print_result(cls.start_time)
447 cleanup.delete_temps() 445 cleanup.delete_temps()
448 print(_("Exiting...")) 446 print(_("Exiting..."))
447 pool.close()
448 pool.join()
449 exit(0) 449 exit(0)
450 450
451 @classmethod 451 @classmethod
452 def run_test(cls, test, result): 452 def run_test(cls, test, p_bar=None):
453 """Create a new test suite, add a test, and run it 453 """Create a new test suite, add a test, and run it
454 454
455 :param test: The test to add to the suite 455 :param test: The test to add to the suite
456 :param result: The result object to append to 456 :param result: The result object to append to
457 :type result: :class:`syntribos.result.IssueTestResult` 457 :type result: :class:`syntribos.result.IssueTestResult`
458 :param bool dry_run: (OPTIONAL) Only print out test names
459 """ 458 """
460 suite = unittest.TestSuite() 459 if test:
461 suite.addTest(test("run_test_case")) 460 suite = unittest.TestSuite()
462 suite.run(result) 461 suite.addTest(test("run_test_case"))
462 suite.run(result)
463 if p_bar:
464 with lock:
465 p_bar.increment(1)
466 p_bar.print_bar()
463 467
464 468
465def entry_point(): 469def entry_point():
diff --git a/syntribos/tests/base.py b/syntribos/tests/base.py
index d91a7a9..cff112b 100644
--- a/syntribos/tests/base.py
+++ b/syntribos/tests/base.py
@@ -148,10 +148,11 @@ class BaseTestCase(unittest.TestCase):
148 :param str filename: name of template file 148 :param str filename: name of template file
149 :param str file_content: content of template file as string 149 :param str file_content: content of template file as string
150 """ 150 """
151 cls.init_req = parser.create_request( 151 if not cls.init_req:
152 file_content, CONF.syntribos.endpoint, meta_vars) 152 cls.init_req = parser.create_request(
153 153 file_content, CONF.syntribos.endpoint, meta_vars)
154 prepared_copy = cls.init_req.get_prepared_copy() 154 prepared_copy = cls.init_req.get_prepared_copy()
155 cls.prepared_init_req = prepared_copy
155 cls.init_resp, cls.init_signals = cls.client.send_request( 156 cls.init_resp, cls.init_signals = cls.client.send_request(
156 prepared_copy) 157 prepared_copy)
157 if cls.init_resp is not None: 158 if cls.init_resp is not None:
diff --git a/syntribos/tests/fuzz/base_fuzz.py b/syntribos/tests/fuzz/base_fuzz.py
index 34778c4..0581c1a 100644
--- a/syntribos/tests/fuzz/base_fuzz.py
+++ b/syntribos/tests/fuzz/base_fuzz.py
@@ -67,6 +67,9 @@ class BaseFuzzTestCase(base.BaseTestCase):
67 headers=cls.request.headers, 67 headers=cls.request.headers,
68 params=cls.request.params, 68 params=cls.request.params,
69 data=cls.request.data) 69 data=cls.request.data)
70
71 if not hasattr(cls.request, 'body'):
72 cls.request.body = cls.request.data
70 cls.test_req = cls.request 73 cls.test_req = cls.request
71 74
72 if cls.test_resp is None or "EXCEPTION_RAISED" in cls.test_signals: 75 if cls.test_resp is None or "EXCEPTION_RAISED" in cls.test_signals:
@@ -88,7 +91,6 @@ class BaseFuzzTestCase(base.BaseTestCase):
88 self.run_default_checks() in order to test for the Issues 91 self.run_default_checks() in order to test for the Issues
89 defined here 92 defined here
90 """ 93 """
91
92 if "HTTP_STATUS_CODE_5XX" in self.test_signals: 94 if "HTTP_STATUS_CODE_5XX" in self.test_signals:
93 self.register_issue( 95 self.register_issue(
94 defect_type="500_errors", 96 defect_type="500_errors",
@@ -125,9 +127,6 @@ class BaseFuzzTestCase(base.BaseTestCase):
125 def get_test_cases(cls, filename, file_content): 127 def get_test_cases(cls, filename, file_content):
126 """Generates new TestCases for each fuzz string 128 """Generates new TestCases for each fuzz string
127 129
128 First, sends a baseline (non-fuzzed) request, storing it in
129 cls.init_resp.
130
131 For each string returned by cls._get_strings(), yield a TestCase class 130 For each string returned by cls._get_strings(), yield a TestCase class
132 for the string as an extension to the current TestCase class. Every 131 for the string as an extension to the current TestCase class. Every
133 string used as a fuzz test payload entails the generation of a new 132 string used as a fuzz test payload entails the generation of a new
@@ -197,8 +196,7 @@ class BaseFuzzTestCase(base.BaseTestCase):
197 issue.request = self.test_req 196 issue.request = self.test_req
198 issue.response = self.test_resp 197 issue.response = self.test_resp
199 issue.test_type = self.test_name 198 issue.test_type = self.test_name
200 prepared_copy = self.init_req.get_prepared_copy() 199 url_components = urlparse(self.prepared_init_req.url)
201 url_components = urlparse(prepared_copy.url)
202 issue.target = url_components.netloc 200 issue.target = url_components.netloc
203 issue.path = url_components.path 201 issue.path = url_components.path
204 issue.init_signals = self.init_signals 202 issue.init_signals = self.init_signals
diff --git a/syntribos/tests/fuzz/buffer_overflow.py b/syntribos/tests/fuzz/buffer_overflow.py
index 25f45ba..6ca0565 100644
--- a/syntribos/tests/fuzz/buffer_overflow.py
+++ b/syntribos/tests/fuzz/buffer_overflow.py
@@ -34,7 +34,6 @@ class BufferOverflowBody(base_fuzz.BaseFuzzTestCase):
34 return [ 34 return [
35 "A" * (2 ** 16 + 1), 35 "A" * (2 ** 16 + 1),
36 "a" * 10 ** 5, 36 "a" * 10 ** 5,
37 "a" * 10 ** 6,
38 '\x00' * (2 ** 16 + 1), 37 '\x00' * (2 ** 16 + 1),
39 "%%s" * 513, 38 "%%s" * 513,
40 ] 39 ]
diff --git a/syntribos/utils/cli.py b/syntribos/utils/cli.py
index adec89a..5022bf3 100644
--- a/syntribos/utils/cli.py
+++ b/syntribos/utils/cli.py
@@ -26,22 +26,6 @@ CONF = cfg.CONF
26def print_symbol(): 26def print_symbol():
27 """Syntribos radiation symbol.""" 27 """Syntribos radiation symbol."""
28 symbol = """ Syntribos 28 symbol = """ Syntribos
29 xxxxxxx
30 x xxxxxxxxxxxxx x
31 x xxxxxxxxxxx x
32 xxxxxxxxx
33 x xxxxxxx x
34 xxxxx
35 x xxx x
36 x
37 xxxxxxxxxxxxxxx xxxxxxxxxxxxxxx
38 xxxxxxxxxxxxx xxxxxxxxxxxxx
39 xxxxxxxxxxx xxxxxxxxxxx
40 xxxxxxxxx xxxxxxxxx
41 xxxxxx xxxxxx
42 xxx xxx
43 x x
44 x
45 === Automated API Scanning ===""" 29 === Automated API Scanning ==="""
46 print(syntribos.SEP) 30 print(syntribos.SEP)
47 print(symbol) 31 print(symbol)
@@ -55,15 +39,24 @@ def colorize(string, color="nocolor"):
55 colors = dict(list(zip(color_names, list(range(31, 35))))) 39 colors = dict(list(zip(color_names, list(range(31, 35)))))
56 colors["nocolor"] = 0 # No Color 40 colors["nocolor"] = 0 # No Color
57 41
58 if not CONF.colorize: 42 if CONF.no_colorize:
59 return string 43 return string
60 return "\033[0;{color}m{string}\033[0;m".format(string=string, 44 return "\033[0;{color}m{string}\033[0;m".format(string=string,
61 color=colors.setdefault( 45 color=colors.setdefault(
62 color, 0)) 46 color, 0))
63 47
64 48
49def colorize_by_percent(amount, total, high=0.5, medium=0):
50 if amount > total * high:
51 return colorize(amount, "red")
52 elif amount > total * medium:
53 return colorize(amount, "yellow")
54 else:
55 return str(amount)
56
57
65class ProgressBar(object): 58class ProgressBar(object):
66 """A simple progressBar. 59 """A simple progressBar. Written as a singleton.
67 60
68 A simple generic progress bar like many others. 61 A simple generic progress bar like many others.
69 :param int total_len: total_len value, when progress is 100 % 62 :param int total_len: total_len value, when progress is 100 %
diff --git a/syntribos/utils/config_fixture.py b/syntribos/utils/config_fixture.py
index 762423c..c63e9c3 100644
--- a/syntribos/utils/config_fixture.py
+++ b/syntribos/utils/config_fixture.py
@@ -64,7 +64,7 @@ class ConfFixture(config_fixture.Config):
64 """config values for CLI options(default group).""" 64 """config values for CLI options(default group)."""
65 # TODO(unrahul): Add mock file path for outfile 65 # TODO(unrahul): Add mock file path for outfile
66 self.conf.set_default("test_types", [""]) 66 self.conf.set_default("test_types", [""])
67 self.conf.set_default("colorize", False) 67 self.conf.set_default("no_colorize", True)
68 self.conf.set_default("output_format", "json") 68 self.conf.set_default("output_format", "json")
69 self.conf.set_default("min_severity", "LOW") 69 self.conf.set_default("min_severity", "LOW")
70 self.conf.set_default("min_confidence", "LOW") 70 self.conf.set_default("min_confidence", "LOW")
diff --git a/tests/unit/test_ascii_colors.py b/tests/unit/test_ascii_colors.py
index 3b3d8eb..83f352b 100644
--- a/tests/unit/test_ascii_colors.py
+++ b/tests/unit/test_ascii_colors.py
@@ -20,7 +20,7 @@ from syntribos.utils.cli import CONF
20class TestColorize(testtools.TestCase): 20class TestColorize(testtools.TestCase):
21 21
22 def test_colorize(self): 22 def test_colorize(self):
23 CONF.colorize = True 23 CONF.no_colorize = False
24 string = "color this string" 24 string = "color this string"
25 colors = {"red": 31, 25 colors = {"red": 31,
26 "green": 32, 26 "green": 32,
@@ -34,6 +34,6 @@ class TestColorize(testtools.TestCase):
34 colorize(string, color)) 34 colorize(string, color))
35 35
36 def test_no_colorize(self): 36 def test_no_colorize(self):
37 CONF.colorize = False 37 CONF.no_colorize = True
38 string = "No color" 38 string = "No color"
39 self.assertEqual(string, colorize(string)) 39 self.assertEqual(string, colorize(string))
diff --git a/tests/unit/test_results.py b/tests/unit/test_results.py
index e6fe8c6..d80a0d6 100644
--- a/tests/unit/test_results.py
+++ b/tests/unit/test_results.py
@@ -52,7 +52,7 @@ class TestIssueTestResult(testtools.TestCase):
52 def test_addFailure(self): 52 def test_addFailure(self):
53 test = FakeTest("failure") 53 test = FakeTest("failure")
54 self.issue_result.addFailure(test, ()) 54 self.issue_result.addFailure(test, ())
55 self.assertEqual(self.issue_result.stats["failures"], 2) 55 self.assertEqual(self.issue_result.stats["unique_failures"], 2)
56 56
57 def test_addSuccess(self): 57 def test_addSuccess(self):
58 test = FakeTest("success") 58 test = FakeTest("success")