Fix various pep8 violations

export_static.py

@@ -12,16 +12,18 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import os
-import gzip
-import shutil
+from __future__ import print_function
+
 import django
+import gzip
+import os
+import shutil
 
 from argparse import ArgumentParser
 from django.http import Http404
 
-from django.test import RequestFactory
 from django.core.urlresolvers import resolve
+from django.test import RequestFactory
 
 from stackviz.parser import tempest_subunit
 from stackviz import settings
@@ -62,7 +64,7 @@ def export_single_page(path, dest_dir, use_gzip=False):
         with open_func(os.path.join(dest_dir, dest_file), 'wb') as f:
             f.write(content)
     except Http404 as ex:
-        print "Warning: skipping %s due to error: %s" % (path, ex.message)
+        print("Warning: skipping %s due to error: %s" % (path, ex.message))
 
 
 def init_django(args):
@@ -105,35 +107,35 @@ def main():
 
     if not args.ignore_bower:
         if not os.listdir(os.path.join('stackviz', 'static', 'components')):
-            print "Bower components have not been installed, please run " \
-                  "`bower install`"
+            print("Bower components have not been installed, please run "
+                  "`bower install`")
             return 1
 
     if os.path.exists(args.path):
         if os.listdir(args.path):
-            print "Destination exists and is not empty, cannot continue"
+            print("Destination exists and is not empty, cannot continue")
             return 1
 
         os.mkdir(args.path)
 
     init_django(args)
 
-    print "Copying static files ..."
+    print("Copying static files ...")
     shutil.copytree(os.path.join('stackviz', 'static'),
                     os.path.join(args.path, 'static'))
 
     for path in EXPORT_PATHS:
-        print "Rendering:", path
+        print("Rendering:", path)
         export_single_page(path, args.path)
 
     repos = tempest_subunit.get_repositories()
     if repos:
         for run_id in range(repos[0].count()):
-            print "Rendering views for tempest run #%d" % run_id
+            print("Rendering views for tempest run #%d" % (run_id))
             export_single_page('/tempest_timeline_%d.html' % run_id, args.path)
             export_single_page('/tempest_results_%d.html' % run_id, args.path)
 
-            print "Exporting data for tempest run #%d" % run_id
+            print("Exporting data for tempest run #%d" % (run_id))
             export_single_page('/tempest_api_tree_%d.json' % run_id,
                                args.path, args.gzip)
             export_single_page('/tempest_api_raw_%d.json' % run_id,
@@ -141,10 +143,10 @@ def main():
             export_single_page('/tempest_api_details_%d.json' % run_id,
                                args.path, args.gzip)
     else:
-        print "Warning: no test repository could be loaded, no data will be " \
-              "available!"
+        print("Warning: no test repository could be loaded, no data will "
+              "be available!")
 
-    print "Exporting DStat log: dstat_log.csv"
+    print("Exporting DStat log: dstat_log.csv")
     export_single_page('/dstat_log.csv', args.path, args.gzip)
 
 

stackviz/global_template_injector.py

@@ -15,6 +15,7 @@
 from stackviz.parser.tempest_subunit import get_repositories
 from stackviz.settings import USE_GZIP
 
+
 def inject_extra_context(request):
     ret = {
         'use_gzip': USE_GZIP

stackviz/parser/devstack_parser.py

@@ -33,7 +33,8 @@
 
 import os
 
-from datetime import datetime, timedelta
+from datetime import datetime
+from datetime import timedelta
 from log_node import LogNode
 
 #: The format of the timestamp prefixing each log entry
@@ -41,6 +42,7 @@ TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S.%f'
 
 
 def extract_date(line):
+
     """
     Extracts a date from the given line, returning the parsed date and
     remaining contents of the line.
@@ -48,6 +50,7 @@ def extract_date(line):
     :param line: the line to extract a date from
     :return: a tuple of the parsed date and remaining line contents
     """
+
     date_str, message = line.split(' | ', 1)
     date = datetime.strptime(date_str, TIMESTAMP_FORMAT)
 
@@ -55,6 +58,7 @@ def extract_date(line):
 
 
 def parse_summary(summary_path):
+
     """
     Parses a summary logfile. Summary entries are prefixed with identical
     datestamps to those in the main log, but have only explicit log messages
@@ -66,6 +70,7 @@ def parse_summary(summary_path):
     :param summary_path: the path to the summary file to parse
     :return: a list of ordered `LogNode` instances
     """
+
     ret = []
 
     last_node = None
@@ -85,6 +90,7 @@ def parse_summary(summary_path):
 
 
 def parse_log(log_path):
+
     """
     Parses a general `stack.sh` logfile, forming a full log tree based on the
     hierarchy of nested commands as presented in the log.
@@ -96,6 +102,7 @@ def parse_log(log_path):
     :param log_path: the path to the logfile to parse
     :return: a list of parsed `LogNode` instances
     """
+
     last_depth = 1
     last_node = None
 
@@ -144,6 +151,7 @@ def parse_log(log_path):
 
 
 def merge(summary, log):
+
     """
     Merges general log entries into parent categories based on their timestamp
     relative to the summary output timestamp.
@@ -157,6 +165,7 @@ def merge(summary, log):
     :param log: the list of general log nodes
     :return: the original summary nodes with children set to the log nodes
     """
+
     if not summary:
         return []
 
@@ -180,6 +189,7 @@ def merge(summary, log):
 
 
 def bootstrap(log_path, summary_path=None):
+
     """
     Loads, parses, and merges the given log and summary files. The path to the
     summary file will be determined automatically based on the path to the
@@ -193,6 +203,7 @@ def bootstrap(log_path, summary_path=None):
     :return: a list of merged `LogNode` instances, or `None` if no matching
              summary file can be located automatically
     """
+
     if summary_path:
         return merge(parse_summary(summary_path), parse_log(log_path))
 
@@ -244,4 +255,3 @@ def get_command_totals(node, totals=None):
         totals[combined] += entry.duration_self
 
     return totals
-

stackviz/parser/log_node.py

@@ -12,15 +12,18 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from datetime import datetime
+from datetime import timedelta
+
 from inspect import getmembers
 from numbers import Number
-from datetime import datetime, timedelta
 
 #: The default cutoff for log entries when pruning takes place, in seconds
 DEFAULT_PRUNE_CUTOFF = 0.05
 
 
-class LogNode:
+class LogNode(object):
+
     """
     Represents an entry in an ordered event log, consisting of a date, message,
     and an arbitrary set of child nodes.
@@ -41,10 +44,12 @@ class LogNode:
 
     @property
     def duration(self):
+
         """
         Determines the overall duration for this node, beginning at this parent
         node's start time through the final child's ending time.
         """
+
         if self.children:
             last_sibling = self.children[-1].next_sibling
             if not last_sibling:
@@ -69,6 +74,7 @@ class LogNode:
         return self.next_sibling.date - self.date
 
     def traverse(self):
+
         """
         A generator that will traverse all child nodes of this log tree
         sequentially.
@@ -155,17 +161,18 @@ class LogNode:
 
 
 def prune(nodes, cutoff=DEFAULT_PRUNE_CUTOFF, fill=None):
+
     """
     Prunes the given list of `LogNode` instances, removing nodes whose duration
     is less than the given cutoff value. If a `fill` value is provided, removed
     nodes will be replaced with a single filler value accounting for the lost
-    duration. This filler value will be inserted at the end of the list and will
-    not be properly linked to other values.
+    duration. This filler value will be inserted at the end of the list and
+    will not be properly linked to other values.
 
     Note that returned values will not necessarily be a continuous list of
-    nodes. The original list will remain unchanged; sibling and child references
-    will not be modified to point to account any modified, removed, or added
-    nodes.
+    nodes. The original list will remain unchanged; sibling and child
+    references will not be modified to point to account any modified, removed,
+    or added nodes.
 
     :param nodes: the list of log nodes to prune
     :type nodes: list[LogNode]
@@ -176,6 +183,7 @@ def prune(nodes, cutoff=DEFAULT_PRUNE_CUTOFF, fill=None):
     :type cutoff: float
     :return: a (potentially) reduced list of nodes
     """
+
     ret = []
 
     fill_amount = 0.0

stackviz/parser/tempest_subunit.py

@@ -17,13 +17,13 @@ import re
 from functools import partial
 
 from subunit import ByteStreamToStreamResult
-from testtools import (StreamResult, StreamSummary,
-                       StreamToDict, CopyStreamResult)
+from testtools import CopyStreamResult
+from testtools import StreamResult
+from testtools import StreamSummary
+from testtools import StreamToDict
 
-from testrepository.repository import AbstractTestRun
-from testrepository.repository.file import (RepositoryFactory,
-                                            Repository,
-                                            RepositoryNotFound)
+from testrepository.repository.file import RepositoryFactory
+from testrepository.repository.file import RepositoryNotFound
 
 from stackviz import settings
 
@@ -33,6 +33,7 @@ NAME_TAGS_PATTERN = re.compile(r'^(.+)\[(.+)\]$')
 
 
 def get_repositories():
+
     """
     Loads all test repositories from locations configured in
     `settings.TEST_REPOSITORIES`. Only locations with a valid `.testrepository`
@@ -41,6 +42,7 @@ def get_repositories():
     :return: a list of loaded :class:`Repository` instances
     :rtype: list[Repository]
     """
+
     factory = RepositoryFactory()
 
     ret = []
@@ -48,7 +50,7 @@ def get_repositories():
     for path in settings.TEST_REPOSITORIES:
         try:
             ret.append(factory.open(path))
-        except (ValueError, RepositoryNotFound) as ex:
+        except (ValueError, RepositoryNotFound):
             # skip
             continue
 
@@ -56,7 +58,7 @@ def get_repositories():
 
 
 def _clean_name(name):
-    # TODO: currently throwing away other info - any worth keeping?
+    # TODO(currently throwing away other info - any worth keeping?)
     m = NAME_TAGS_PATTERN.match(name)
     if m:
         # tags = m.group(2).split(',')
@@ -94,16 +96,18 @@ def _read_test(test, out, strip_details):
 
 
 def convert_run(test_run, strip_details=False):
+
     """
-    Converts the given test run into a raw list of test dicts, using the subunit
-    stream as an intermediate format.
+    Converts the given test run into a raw list of test dicts, using the
+    subunit stream as an intermediate format.(see: read_subunit.py from
+    subunit2sql)
 
     :param test_run: the test run to convert
     :type test_run: AbstractTestRun
     :param strip_details: if True, remove test details (e.g. stdout/stderr)
     :return: a list of individual test results
     """
-    # see: read_subunit.py from subunit2sql
+
     ret = []
 
     stream = test_run.get_subunit_stream()
@@ -150,6 +154,7 @@ def _descend_recurse(parent, parts_remaining):
 
 
 def _descend(root, path):
+
     """
     Retrieves the node within the `root` dict denoted by the series of
     '.'-separated children as specified in `path`. Children for each node must
@@ -164,6 +169,7 @@ def _descend(root, path):
     :type path: str
     :return: the dict node representing the last child
     """
+
     path_parts = path.split('.')
     path_parts.reverse()
 
@@ -173,6 +179,7 @@ def _descend(root, path):
 
 
 def reorganize(converted_test_run):
+
     """
     Reorganizes and categorizes the given test run, forming tree of tests
     categorized by their module paths.
@@ -180,6 +187,7 @@ def reorganize(converted_test_run):
     :param converted_test_run:
     :return: a dict tree of test nodes, organized by module path
     """
+
     ret = {}
 
     for entry in converted_test_run:

stackviz/urls.py

@@ -12,8 +12,9 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from django.conf.urls import patterns, include, url
-from django.contrib import admin
+from django.conf.urls import include
+from django.conf.urls import patterns
+from django.conf.urls import url
 
 from stackviz.views.index import IndexView
 
@@ -23,6 +24,6 @@ urlpatterns = patterns(
     url(r'^index.html$', IndexView.as_view(), name="index"),
     url(r'^tempest_', include('stackviz.views.tempest.urls')),
     url(r'^devstack_', include('stackviz.views.devstack.urls')),
-    url(r'^upstream_', include ('stackviz.views.upstream.urls')),
-    url(r'^dstat_', include ('stackviz.views.dstat.urls'))
+    url(r'^upstream_', include('stackviz.views.upstream.urls')),
+    url(r'^dstat_', include('stackviz.views.dstat.urls'))
 )

stackviz/views/devstack/results.py

@@ -14,5 +14,6 @@
 
 from django.views.generic import TemplateView
 
+
 class ResultsView(TemplateView):
     template_name = 'devstack/results.html'

stackviz/views/devstack/urls.py

@@ -12,16 +12,12 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from django.conf.urls import patterns, include, url
-from django.contrib import admin
+from django.conf.urls import patterns
+from django.conf.urls import url
 
 from stackviz.views.devstack.results import ResultsView
 
 
 urlpatterns = patterns('',
-    # Examples:
-    # url(r'^$', 'stackviz.views.home', name='home'),
-    # url(r'^blog/', include('blog.urls')),
-
-    url(r'^results$', ResultsView.as_view()),
-)
+                       url(r'^results$', ResultsView.as_view()),
+                       )

stackviz/views/dstat/api.py

@@ -12,9 +12,9 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import os
+from django.http import Http404
+from django.http import HttpResponse
 
-from django.http import HttpResponse, Http404
 from django.views.generic import View
 
 from stackviz import settings

stackviz/views/dstat/urls.py

@@ -12,10 +12,10 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from django.conf.urls import patterns, include, url
+from django.conf.urls import patterns
+from django.conf.urls import url
 
-from .api import DStatCSVEndpoint
+from api import DStatCSVEndpoint
 
-urlpatterns = patterns('',
-    url(r'^log.csv$', DStatCSVEndpoint.as_view()),
-)
+
+urlpatterns = patterns('', url(r'^log.csv$', DStatCSVEndpoint.as_view()))

stackviz/views/index.py

@@ -14,5 +14,6 @@
 
 from django.views.generic import TemplateView
 
+
 class IndexView(TemplateView):
     template_name = 'index.html'

stackviz/views/tempest/aggregate.py

@@ -14,5 +14,6 @@
 
 from django.views.generic import TemplateView
 
+
 class AggregateResultsView(TemplateView):
-    template_name = 'tempest/aggregate.html'
+    template_name = 'tempest/aggregate.html'

stackviz/views/tempest/api.py

@@ -15,9 +15,9 @@
 from django.http import Http404
 from restless.views import Endpoint
 
-from stackviz.parser.tempest_subunit import (get_repositories,
-                                             convert_run,
-                                             reorganize)
+from stackviz.parser.tempest_subunit import convert_run
+from stackviz.parser.tempest_subunit import get_repositories
+from stackviz.parser.tempest_subunit import reorganize
 
 #: Cached results from loaded subunit logs indexed by their run number
 _cached_run = {}
@@ -33,9 +33,11 @@ _cached_details = {}
 class NoRunDataException(Http404):
     pass
 
+
 class RunNotFoundException(Http404):
     pass
 
+
 class TestNotFoundException(Http404):
     pass
 
@@ -53,7 +55,7 @@ def _load_run(run_id):
         run = repos[0].get_test_run(run_id)
 
         # strip details for now
-        # TODO: provide method for getting details on demand
+        # TODO(provide method for getting details on demand)
         # (preferably for individual tests to avoid bloat)
         converted_run = convert_run(run, strip_details=True)
         _cached_run[run_id] = converted_run
@@ -118,4 +120,3 @@ class TempestRunTreeEndpoint(Endpoint):
 class TempestRunDetailsEndpoint(Endpoint):
     def get(self, request, run_id, test_name=None):
         return _load_details(run_id, test_name)
-

stackviz/views/tempest/results.py

@@ -12,11 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from django.core.urlresolvers import reverse
-from django.views.generic import TemplateView, RedirectView
-from django.http import Http404
-
-from stackviz.parser.tempest_subunit import get_repositories
+from django.views.generic import TemplateView
 
 
 class ResultsView(TemplateView):
@@ -27,4 +23,3 @@ class ResultsView(TemplateView):
         context['run_id'] = self.kwargs['run_id']
 
         return context
-

stackviz/views/tempest/timeline.py

@@ -12,11 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from django.core.urlresolvers import reverse
-from django.views.generic import TemplateView, RedirectView
-from django.http import Http404
-
-from stackviz.parser.tempest_subunit import get_repositories
+from django.views.generic import TemplateView
 
 
 class TimelineView(TemplateView):
@@ -27,4 +23,3 @@ class TimelineView(TemplateView):
         context['run_id'] = self.kwargs['run_id']
 
         return context
-

stackviz/views/tempest/urls.py

@@ -12,15 +12,17 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from django.conf.urls import patterns, include, url
 
-from .results import ResultsView
-from .timeline import TimelineView
-from .aggregate import AggregateResultsView
+from django.conf.urls import patterns
+from django.conf.urls import url
 
-from .api import (TempestRunTreeEndpoint,
-                  TempestRunRawEndpoint,
-                  TempestRunDetailsEndpoint)
+from aggregate import AggregateResultsView
+from results import ResultsView
+from timeline import TimelineView
+
+from api import TempestRunDetailsEndpoint
+from api import TempestRunRawEndpoint
+from api import TempestRunTreeEndpoint
 
 
 urlpatterns = patterns('',

stackviz/views/upstream/api.py

@@ -12,30 +12,34 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from subunit2sql.db import api, models
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.orm.state import InstanceState
 from restless.views import Endpoint
 
-import json
+from subunit2sql.db import api
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
 
 def _get_runs(change_id):
+
     '''
     When given the change_id of a Gerrit change, a connection will be made to
     the upstream subunit2sql db and query all run meta having that change_id
     :param change_id: the Gerrit change_id to query
     :return: a json dict of run_meta objects
     '''
-    engine=create_engine('mysql://query:query@logstash.openstack.org:3306/subunit2sql')
+
+    engine = create_engine('mysql://query:query@logstash.openstack.org' +
+                           ':3306/subunit2sql')
 
     Session = sessionmaker(bind=engine)
 
     # create a Session
     session = Session()
 
-    list_of_runs = api.get_runs_by_key_value(key="build_change",value=change_id,
-                                     session=session)
+    list_of_runs = api.get_runs_by_key_value(key="build_change",
+                                             value=change_id,
+                                             session=session)
     ret_list = []
 
     for run in list_of_runs:
@@ -47,10 +51,12 @@ def _get_runs(change_id):
 class GerritURLEndpoint(Endpoint):
 
     def get(self, request, change_id):
+
         '''
         :param request:
         :param change_id:
         :return: Collection of run objects associated with a
                  specific CID
         '''
+
         return _get_runs(change_id)

stackviz/views/upstream/run.py

@@ -14,17 +14,6 @@
 
 from django.views.generic import TemplateView
 
-# TODO Planned f(x):
-# 1. Input a run_id from an upstream run to display run info
-# 2. Compare runs by metadata
-#
-# 1.
-#   EX: url for logstash=
-# http://logs.openstack.org/92/206192/2/check/gate-subunit2sql-python27/c1ff374/
-#
-#   a. link between logstash and subunit2sql (urlparser)
-#   b. display server-side as well as client-side logs
-#
+
 class RunView(TemplateView):
     template_name = 'upstream/run.html'
-

stackviz/views/upstream/test.py

@@ -14,9 +14,10 @@
 
 from django.views.generic import TemplateView
 
-# TODO: Planned f(x):
+# TODO(Planned functionality)
 # Compare one specific test against its moving average
 #
 
+
 class TestView(TemplateView):
-    template_name = 'upstream/test.html'
+    template_name = 'upstream/test.html'

stackviz/views/upstream/urls.py

@@ -12,12 +12,13 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from django.conf.urls import patterns, include, url
+from django.conf.urls import patterns
+from django.conf.urls import url
 
-from .run import RunView
-from .test import TestView
+from run import RunView
+from test import TestView
 
-from .api import GerritURLEndpoint
+from api import GerritURLEndpoint
 
 urlpatterns = patterns('',