Exit with nonzero return code if Elastic indexing fails

At the end of a Browbeat run loop through all the tests and their results
to check if any failed to index, if so set the return code to 1.

This way tests will complete even if indexing fails but scripts and other
software checking the return code will fail out properly instead of allowing
data to be lost silently.

Change-Id: If1a7b2b41df35abbb8a6cc63d6003c1ae36664dd

browbeat.py

@@ -111,11 +111,22 @@ def main():
         _logger.info("Saved browbeat result summary to {}".format(
             os.path.join(result_dir,time_stamp + '.' + 'report')))
         lib.WorkloadBase.WorkloadBase.print_summary()
+
+        browbeat_rc = 0
         if lib.WorkloadBase.WorkloadBase.failure > 0:
-           _logger.info("Browbeat Finished with Failures, UUID: {}".format(browbeat_uuid))
-           sys.exit(1)
+            browbeat_rc = 1
+        if lib.WorkloadBase.WorkloadBase.index_failures > 0:
+            browbeat_rc = 2
+
+        if browbeat_rc == 1:
+           _logger.info("Browbeat finished with test failures, UUID: {}".format(browbeat_uuid))
+           sys.exit(browbeat_rc)
+        elif browbeat_rc == 2:
+           _logger.info("Browbeat finished with Elasticsearch indexing failures, UUID: {}"
+                        .format(browbeat_uuid))
+           sys.exit(browbeat_rc)
         else:
-           _logger.info("Browbeat Finished Successfully, UUID: {}".format(browbeat_uuid))
+           _logger.info("Browbeat finished successfully, UUID: {}".format(browbeat_uuid))
            sys.exit(0)
 
 if __name__ == '__main__':

lib/PerfKit.py

@@ -104,6 +104,7 @@ class PerfKit(WorkloadBase.WorkloadBase):
                     if not self.elastic.index_result(result, test_name, result_dir,
                                                      str(result_count), 'result'):
                         index_success = False
+                        self.update_index_failures()
         else:
             complete_result_json = {'browbeat_scenario': benchmark_config}
             complete_result_json['perfkit_errors'] = self.get_error_details(result_dir)

lib/Rally.py

@@ -351,6 +351,8 @@ class Rally(WorkloadBase.WorkloadBase):
                                             # Start indexing
                                             index_status = self.json_result(
                                                 task_id, scenario_name, run, test_name, result_dir)
+                                            if not index_status:
+                                                self.update_index_failures()
                                             self.get_time_dict(to_time, from_time,
                                                                benchmark[
                                                                    'name'], new_test_name,

lib/Shaker.py

@@ -126,6 +126,7 @@ class Shaker(WorkloadBase.WorkloadBase):
             result = self.elastic.combine_metadata(shaker_stats)
             index_status = self.elastic.index_result(result, test_name, result_dir, _type='error')
             if index_status is False:
+                self.update_index_failures()
                 return False
             else:
                 return True

lib/WorkloadBase.py

@@ -23,6 +23,7 @@ class WorkloadBase(object):
     failure = 0
     total_tests = 0
     total_scenarios = 0
+    index_failures = 0
     browbeat = {}
 
     @abc.abstractmethod
@@ -53,6 +54,9 @@ class WorkloadBase(object):
     def update_total_fail_tests(self):
         WorkloadBase.failure += 1
 
+    def update_index_failures(self):
+        WorkloadBase.index_failures += 1
+
     def workload_logger(self, result_dir, workload):
         base = result_dir.split('/')
         if not os.path.isfile("{}/{}/browbeat-{}-run.log".format(base[0], base[1], workload)):