Additional baseline candidate test coverage

Additional test cases to improve functional test coverage for baseline
candidate tests. Also includes example files for more complex candidate
comparisons.

Change-Id: I7d05c0f1e3f0d9e0a36c825654d28c19ed4c0bbd

examples/new_candidates-none.py

@@ -0,0 +1,8 @@
+def subprocess_shell_cmd():
+    # sample function with known subprocess shell cmd candidates
+
+def yaml_load():
+    # sample function with known yaml.load candidates
+
+def xml_sax_make_parser():
+    # sample function with known xml.sax.make_parser candidates

examples/new_candidates-nosec.py

@@ -0,0 +1,18 @@
+import xml
+import yaml
+
+def subprocess_shell_cmd():
+    # sample function with known subprocess shell cmd candidates
+    # candidate #2
+    subprocess.Popen('/bin/ls *', shell=True) # nosec
+
+def yaml_load():
+    # sample function with known yaml.load candidates
+    temp_str = yaml.dump({'a': '1', 'b': '2'})
+    # candidate #4
+    y = yaml.load(temp_str) # nosec
+
+def xml_sax_make_parser():
+    # sample function with known xml.sax.make_parser candidates
+    # candidate #6
+    xml.sax.make_parser() # nosec

examples/new_candidates-some.py

@@ -0,0 +1,20 @@
+import xml
+import yaml
+
+def subprocess_shell_cmd():
+    # sample function with known subprocess shell cmd candidates
+    # candidate #1
+    subprocess.Popen('/bin/ls *', shell=True)
+    # candidate #2
+    subprocess.Popen('/bin/ls *', shell=True) # nosec
+
+def yaml_load():
+    # sample function with known yaml.load candidates
+    temp_str = yaml.dump({'a': '1', 'b': '2'})
+    # candidate #4
+    y = yaml.load(temp_str) # nosec
+
+def xml_sax_make_parser():
+    # sample function with known xml.sax.make_parser candidates
+    # candidate #6
+    xml.sax.make_parser() # nosec

tests/functional/test_baseline.py

@@ -19,6 +19,22 @@ import subprocess
 import fixtures
 import testtools
 
+new_candidates_all_total_lines = "Total lines of code: 12"
+new_candidates_some_total_lines = "Total lines of code: 9"
+new_candidates_no_nosec_lines = "Total lines skipped (#nosec): 0"
+new_candidates_skip_nosec_lines = "Total lines skipped (#nosec): 3"
+baseline_no_skipped_files = "Files skipped (0):"
+baseline_no_issues_found = "No issues identified."
+xml_sax_issue_id = "Issue: [B317:blacklist]"
+yaml_load_issue_id = "Issue: [B506:yaml_load]"
+shell_issue_id = "Issue: [B602:subprocess_popen_with_shell_equals_true]"
+candidate_example_one = "subprocess.Popen('/bin/ls *', shell=True)"
+candidate_example_two = "subprocess.Popen('/bin/ls *', shell=True) # nosec"
+candidate_example_three = "y = yaml.load(temp_str)"
+candidate_example_four = "y = yaml.load(temp_str) # nosec"
+candidate_example_five = "xml.sax.make_parser()"
+candidate_example_six = "xml.sax.make_parser() # nosec"
+
 
 class BaselineFunctionalTests(testtools.TestCase):
 
@@ -109,10 +125,10 @@ class BaselineFunctionalTests(testtools.TestCase):
                                      target_directory, baseline_report))
         # assert there were no results (no candidates found)
         self.assertEqual(0, return_code)
-        self.assertIn("Total lines of code: 12", return_value)
-        self.assertIn("Total lines skipped (#nosec): 3", return_value)
-        self.assertIn("Files skipped (0):", return_value)
-        self.assertIn("No issues identified.", return_value)
+        self.assertIn(new_candidates_all_total_lines, return_value)
+        self.assertIn(new_candidates_skip_nosec_lines, return_value)
+        self.assertIn(baseline_no_skipped_files, return_value)
+        self.assertIn(baseline_no_issues_found, return_value)
 
     def test_no_existing_no_new_candidates(self):
         '''Tests when there are no new or existing candidates
@@ -132,6 +148,162 @@ class BaselineFunctionalTests(testtools.TestCase):
         # assert there were no results (no candidates found)
         self.assertEqual(0, return_code)
         self.assertIn("Total lines of code: 1", return_value)
-        self.assertIn("Total lines skipped (#nosec): 0", return_value)
-        self.assertIn("Files skipped (0):", return_value)
-        self.assertIn("No issues identified.", return_value)
+        self.assertIn(new_candidates_no_nosec_lines, return_value)
+        self.assertIn(baseline_no_skipped_files, return_value)
+        self.assertIn(baseline_no_issues_found, return_value)
+
+    def test_no_existing_with_new_candidates(self):
+        '''Tests when there are new candidates and no existing candidates
+
+        Test that bandit returns issues found in file that had no existing
+        candidates from baseline but now contain candidates.
+        '''
+        baseline_report_files = {"new_candidates-all.py":
+                                 "new_candidates-none.py"}
+        target_directory, baseline_code = (self._create_baseline(
+                                           baseline_report_files))
+        # assert the initial baseline found nothing
+        self.assertEqual(0, baseline_code)
+        baseline_report = os.path.join(target_directory,
+                                       self.baseline_report_file)
+        return_value, return_code = (self._run_bandit_baseline(
+                                     target_directory, baseline_report))
+        # assert there were results (candidates found)
+        self.assertEqual(1, return_code)
+        self.assertIn(new_candidates_all_total_lines, return_value)
+        self.assertIn(new_candidates_skip_nosec_lines, return_value)
+        self.assertIn(baseline_no_skipped_files, return_value)
+        self.assertIn(xml_sax_issue_id, return_value)
+        self.assertIn(yaml_load_issue_id, return_value)
+        self.assertIn(shell_issue_id, return_value)
+        # candidate #1
+        self.assertIn(candidate_example_one, return_value)
+        # candidate #3
+        self.assertIn(candidate_example_three, return_value)
+        # candidate #5
+        self.assertIn(candidate_example_five, return_value)
+
+    def test_existing_and_new_candidates(self):
+        '''Tests when tere are new candidates and existing candidates
+
+        Test that bandit returns issues found in file with existing
+        candidates. The new candidates should be returned in this case.
+        '''
+        baseline_report_files = {"new_candidates-all.py":
+                                 "new_candidates-some.py"}
+        target_directory, baseline_code = (self._create_baseline(
+                                           baseline_report_files))
+        # assert the initial baseline found results
+        self.assertEqual(1, baseline_code)
+        baseline_report = os.path.join(target_directory,
+                                       self.baseline_report_file)
+        return_value, return_code = (self._run_bandit_baseline(
+                                     target_directory, baseline_report))
+        # assert there were results (candidates found)
+        self.assertEqual(1, return_code)
+        self.assertIn(new_candidates_all_total_lines, return_value)
+        self.assertIn(new_candidates_skip_nosec_lines, return_value)
+        self.assertIn(baseline_no_skipped_files, return_value)
+        self.assertIn(xml_sax_issue_id, return_value)
+        self.assertIn(yaml_load_issue_id, return_value)
+        # candidate #3
+        self.assertIn(candidate_example_three, return_value)
+        # candidate #5
+        self.assertIn(candidate_example_five, return_value)
+
+    def test_no_new_candidates_include_nosec(self):
+        '''Test to check nosec references with no new candidates
+
+        Test that nosec references are included during a baseline test, which
+        would normally be ignored. In this test case, there are no new
+        candidates even while including the nosec references.
+        '''
+        self.baseline_commands.append('--ignore-nosec')
+        baseline_report_files = {"new_candidates-all.py":
+                                 "new_candidates-all.py"}
+        target_directory, baseline_code = (self._create_baseline(
+                                           baseline_report_files))
+        # assert the intial baseline found results
+        self.assertEqual(1, baseline_code)
+        baseline_report = os.path.join(target_directory,
+                                       self.baseline_report_file)
+        return_value, return_code = (self._run_bandit_baseline(
+                                     target_directory, baseline_report))
+        # assert there were no results (candidates found)
+        self.assertEqual(0, return_code)
+        self.assertIn(new_candidates_all_total_lines, return_value)
+        self.assertIn(new_candidates_no_nosec_lines, return_value)
+        self.assertIn(baseline_no_skipped_files, return_value)
+        self.assertIn(baseline_no_issues_found, return_value)
+
+    def test_new_candidates_include_nosec_only_nosecs(self):
+        '''Test to check nosec references with new only nosec candidates
+
+        Test that nosec references are included during a baseline test, which
+        would normally be ignored. In this test case, there are new candidates
+        which are specifically nosec references.
+        '''
+        self.baseline_commands.append('--ignore-nosec')
+        baseline_report_files = {"new_candidates-nosec.py":
+                                 "new_candidates-none.py"}
+        target_directory, baseline_code = (self._create_baseline(
+                                           baseline_report_files))
+        # assert the initial baseline found nothing
+        self.assertEqual(0, baseline_code)
+        baseline_report = os.path.join(target_directory,
+                                       self.baseline_report_file)
+        return_value, return_code = (self._run_bandit_baseline(
+                                     target_directory, baseline_report))
+        # assert there were results (candidates found)
+        self.assertEqual(1, return_code)
+        self.assertIn(new_candidates_some_total_lines, return_value)
+        self.assertIn(new_candidates_no_nosec_lines, return_value)
+        self.assertIn(baseline_no_skipped_files, return_value)
+        self.assertIn(xml_sax_issue_id, return_value)
+        self.assertIn(yaml_load_issue_id, return_value)
+        self.assertIn(shell_issue_id, return_value)
+        # candidate #2
+        self.assertIn(candidate_example_two, return_value)
+        # candidate #4
+        self.assertIn(candidate_example_four, return_value)
+        # candidate #6
+        self.assertIn(candidate_example_six, return_value)
+
+    def test_new_candidates_include_nosec_new_nosecs(self):
+        '''Test to check nosec references with new candidates, including nosecs
+
+        Test that nosec references are included during a baseline test, which
+        would normally be ignored. In this test case, there are new candidates
+        that also includes new nosec references as well.
+        '''
+        self.baseline_commands.append('--ignore-nosec')
+        baseline_report_files = {"new_candidates-all.py":
+                                 "new_candidates-none.py"}
+        target_directory, baseline_code = (self._create_baseline(
+                                           baseline_report_files))
+        # assert the initial baseline found nothing
+        self.assertEqual(0, baseline_code)
+        baseline_report = os.path.join(target_directory,
+                                       self.baseline_report_file)
+        return_value, return_code = (self._run_bandit_baseline(
+                                     target_directory, baseline_report))
+        # assert there were results (candidates found)
+        self.assertEqual(1, return_code)
+        self.assertIn(new_candidates_all_total_lines, return_value)
+        self.assertIn(new_candidates_no_nosec_lines, return_value)
+        self.assertIn(baseline_no_skipped_files, return_value)
+        self.assertIn(xml_sax_issue_id, return_value)
+        self.assertIn(yaml_load_issue_id, return_value)
+        self.assertIn(shell_issue_id, return_value)
+        # candidate #1
+        self.assertIn(candidate_example_one, return_value)
+        # candidate #2
+        self.assertIn(candidate_example_two, return_value)
+        # candidate #3
+        self.assertIn(candidate_example_three, return_value)
+        # candidate #4
+        self.assertIn(candidate_example_four, return_value)
+        # candidate #5
+        self.assertIn(candidate_example_five, return_value)
+        # candidate #6
+        self.assertIn(candidate_example_six, return_value)