Optimize communication with Gerrit

Also: 1. Look over comments only, do not use votes (because some drivers are executed by jenkins) 2. For master branch assign release based on date Change-Id: I75bb29c8d8cf798a882e64953c784e01fd57a95e
2015-09-30 19:12:34 +03:00 · 2015-09-30 19:12:34 +03:00 · b6168a8ab6
parent 48b982e4a7
commit b6168a8ab6
6 changed files with 85 additions and 62 deletions
--- a/driverlog/processor/main.py
+++ b/driverlog/processor/main.py
@ -28,18 +28,6 @@ from driverlog.processor import utils
 LOG = logging.getLogger(__name__)


-def _find_vote(review, ci_id):
-    """Finds vote corresponding to ci_id."""
-    for approval in (review['currentPatchSet'].get('approvals') or []):
-        if approval['type'] not in ['Verified', 'VRIF']:
-            continue
-
-        if approval['by'].get('username') == ci_id:
-            return approval['value'] in ['1', '2']
-
-    return None
-
-
 def find_ci_result(review_iterator, ci):
    """For a given stream of reviews finds result left by specified ci."""

@ -51,9 +39,14 @@ def find_ci_result(review_iterator, ci):
                continue

            message = comment['message']
+
+            prefix = 'Patch Set'
+            if comment['message'].find(prefix) != 0:
+                continue  # look for special messages only
+
            prefix = 'Patch Set %s:' % review['currentPatchSet']['number']
            if comment['message'].find(prefix) != 0:
-                break  # all comments from the latest patch set passed
+                break  # all comments from the latest patch set already parsed
            message = message[len(prefix):].strip()

            result = None
@ -62,30 +55,36 @@ def find_ci_result(review_iterator, ci):
            success_pattern = ci.get('success_pattern')
            failure_pattern = ci.get('failure_pattern')

-            if success_pattern and re.search(success_pattern, message):
-                result = True
-            elif failure_pattern and re.search(failure_pattern, message):
-                result = False
+            message_lines = (l for l in message.split('\n') if l.strip())

-            # try to get result from vote
-            if result is None:
-                result = _find_vote(review, ci['id'])
+            line = ''
+            for line in message_lines:
+                if success_pattern and re.search(success_pattern, line):
+                    result = True
+                    break
+                elif failure_pattern and re.search(failure_pattern, line):
+                    result = False
+                    break

            if result is not None:
                return {
                    'ci_result': result,
-                    'comment': message,
+                    'comment': line,
                    'timestamp': comment['timestamp'],
                    'review_url': review_url,
                }


-def _get_release_by_branch(releases, branch):
+def _get_release_by_branch(record_ts, releases, branch):
    """Translates branch name into release_id."""
    release = branch.lower()
    if release.find('/') > 0:
        return release.split('/')[1]
    elif release == 'master':
+        for r in reversed(releases):
+            start = utils.date_to_timestamp(r.get('start'))
+            if record_ts > start:
+                return r['id'].lower()
        return releases[-1]['id'].lower()


@ -94,7 +93,8 @@ def update_drivers(drivers, releases):

    Returns True if info was updated
    """
-    branches = [('stable/' + r['id'].lower()) for r in releases] + ['master']
+    branches = [('stable/' + r['id'].lower()) for r in releases
+                if r.get('active')] + ['master']

    rcs_inst = rcs.get_rcs(cfg.CONF.review_uri)
    rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename,
@ -121,13 +121,17 @@ def update_drivers(drivers, releases):
                                           reviewer=ci_id)
            ci_result = find_ci_result(review_iterator, driver['ci'])
            if ci_result:
-                LOG.debug('Found CI result: %s', ci_result)
+                LOG.info('Found CI result: %s', ci_result)
                has_updates = True

                key = (project_id, driver['vendor'], driver['name'])
-                os_version = _get_release_by_branch(releases, branch)
+                os_version = _get_release_by_branch(ci_result['timestamp'],
+                                                    releases, branch)
                ci_result['ci_tested'] = True
                drivers[key]['releases'][os_version] = ci_result
+            else:
+                LOG.warn('CI result is not found for driver: %s in branch: %s',
+                         driver['name'], branch)

    rcs_inst.close()

--- a/driverlog/processor/rcs.py
+++ b/driverlog/processor/rcs.py
@ -26,6 +26,7 @@ LOG = logging.getLogger(__name__)
 DEFAULT_PORT = 29418
 GERRIT_URI_PREFIX = r'^gerrit:\/\/'
 PAGE_LIMIT = 5
+CHUNK_LIMIT = 2


 class Rcs(object):
@ -110,7 +111,11 @@ class Gerrit(Rcs):
    def log(self, **kwargs):
        sort_key = None

-        while True:
+        # DriverLog looks for merged CRs reviewed by specified gerrit-id
+        # it is possible that CR is already merged, but CI didn't vote yet
+        # that's why we grab PAGE_LIMIT*CHUNK_LIMIT CRs from Gerrit
+
+        for i in range(CHUNK_LIMIT):
            cmd = self._get_cmd(sort_key, **kwargs)
            LOG.debug('Executing command: %s', cmd)
            exec_result = self._exec_command(cmd)
--- a/driverlog/tests/unit/test_data/sample_default_data.json
+++ b/driverlog/tests/unit/test_data/sample_default_data.json
@ -16,15 +16,19 @@
    "releases": [
        {
            "id": "Havana",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Havana"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Havana",
+            "start": "2013-Apr-04"
        },
        {
            "id": "Icehouse",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Icehouse"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Icehouse",
+            "start": "2013-Oct-17"
        },
        {
            "id": "Juno",
-            "wiki": "https://wiki.openstack.org/wiki/Releases"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Juno",
+            "start": "2014-Apr-17",
+            "active": true
        }
    ],
    "drivers": [
@ -41,7 +45,9 @@
            "wiki": "https://wiki.openstack.org/wiki/Arista-neutron-ml2-driver",
            "releases": ["Havana", "Icehouse", "Juno"],
            "ci": {
-                "id": "arista-test"
+                "id": "arista-test",
+                "success_pattern": "SUCCESS",
+                "failure_pattern": "FAILURE"
            }
        },
        {
--- a/driverlog/tests/unit/test_main.py
+++ b/driverlog/tests/unit/test_main.py
@ -37,23 +37,6 @@ class TestMain(testtools.TestCase):
    def setUp(self):
        super(TestMain, self).setUp()

-    def test_process_reviews_ci_vote_and_comment(self):
-        # check that vote and matching comment are found
-
-        result = main.find_ci_result([_read_sample_review()],
-                                     {'id': 'arista-test'})
-
-        self.assertIsNotNone(result, 'CI result should be found')
-
-        expected_record = {
-            'ci_result': True,
-            'comment': 'Verified+1\n\nArista third party testing PASSED '
-                       '[ https://arista.box.com/s/x8z0 ]',
-            'timestamp': 1399478047,
-            'review_url': 'https://review.openstack.org/92468',
-        }
-        self.assertEqual(expected_record, result)
-
    def test_process_reviews_ci_only_comments(self):
        # check that comment is found and parsed correctly

@ -67,7 +50,7 @@ class TestMain(testtools.TestCase):

        expected_record = {
            'ci_result': True,
-            'comment': 'Build succeeded.\n\n- neutron_zuul '
+            'comment': '- neutron_zuul '
                       'http://128.107.233.28:8080/job/neutron_zuul/263 : '
                       'SUCCESS in 18m 52s',
            'timestamp': 1399481091,
--- a/etc/default_data.json
+++ b/etc/default_data.json
@ -36,51 +36,66 @@
    "releases": [
        {
            "id": "Austin",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Austin"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Austin",
+            "start": "2010-Jan-01"
        },
        {
            "id": "Bexar",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Bexar"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Bexar",
+            "start": "2010-Oct-21"
        },
        {
            "id": "Cactus",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Cactus"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Cactus",
+            "start": "2011-Feb-03"
        },
        {
            "id": "Diablo",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Diablo"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Diablo",
+            "start": "2011-Apr-15"
        },
        {
            "id": "Essex",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Essex"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Essex",
+            "start": "2011-Sep-22"
        },
        {
            "id": "Folsom",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Folsom"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Folsom",
+            "start": "2012-Apr-05"
        },
        {
            "id": "Grizzly",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Grizzly"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Grizzly",
+            "start": "2012-Oct-04"
        },
        {
            "id": "Havana",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Havana"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Havana",
+            "start": "2013-Apr-04"
        },
        {
            "id": "Icehouse",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Icehouse"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Icehouse",
+            "start": "2013-Oct-17"
        },
        {
            "id": "Juno",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Juno"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Juno",
+            "start": "2014-Apr-17",
+            "active": true
        },
        {
            "id": "Kilo",
-            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Kilo"
+            "wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Kilo",
+            "start": "2014-Oct-16",
+            "active": true
        },
        {
            "id": "Liberty",
-            "wiki": "https://wiki.openstack.org/wiki/Releases"
+            "wiki": "https://wiki.openstack.org/wiki/Releases",
+            "start": "2015-Apr-30",
+            "active": true
        }
    ],
    "drivers": [
--- a/etc/default_data.schema.json
+++ b/etc/default_data.schema.json
@ -30,9 +30,15 @@
                    },
                    "wiki": {
                        "type": "string"
+                    },
+                    "active": {
+                        "type": "boolean"
+                    },
+                    "start": {
+                        "$ref": "#/definitions/date_format"
                    }
                },
-                "required": ["id", "wiki"]
+                "required": ["id", "wiki", "start"]
            }
        },
        "drivers": {
@ -117,12 +123,16 @@
                    "type": "string"
                }
            },
-            "required": ["id"],
+            "required": ["id", "success_pattern", "failure_pattern"],
            "additionalProperties": false
        },
        "release_id": {
            "type": "string",
            "pattern": "^[A-Z][a-z]+$"
+        },
+        "date_format": {
+            "type": ["string"],
+            "pattern": "^20\\d{2}-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-[0-3]\\d$"
        }
    }
 }