From dcd9605f5e0481838e70f26234c222d69037742a Mon Sep 17 00:00:00 2001 From: Mikhail S Medvedev Date: Tue, 13 Oct 2015 17:27:11 -0500 Subject: [PATCH] Remove ciwatch, moved to openstack-infra/ciwatch A separate ciwatch project [1] has been created with [2]. Commit history was retained. [1] http://git.openstack.org/cgit/openstack-infra/ciwatch/tree/ [2] I1a3dba0ea02a905beb6f7e2f9cce6682b202fbd2 Change-Id: Ibc8cd36d5bd0e9af919ef225d383f650287936d4 --- monitoring/ciwatch/LICENSE | 176 ------------------ monitoring/ciwatch/README.md | 53 ------ monitoring/ciwatch/ci-watch.conf.sample | 18 -- monitoring/ciwatch/ciwatch.wsgi | 17 -- monitoring/ciwatch/ciwatch/__init__.py | 31 --- monitoring/ciwatch/ciwatch/api.py | 87 --------- monitoring/ciwatch/ciwatch/cache.py | 38 ---- monitoring/ciwatch/ciwatch/config.py | 38 ---- monitoring/ciwatch/ciwatch/db.py | 57 ------ monitoring/ciwatch/ciwatch/events.py | 174 ----------------- monitoring/ciwatch/ciwatch/filters.py | 32 ---- monitoring/ciwatch/ciwatch/log.py | 46 ----- monitoring/ciwatch/ciwatch/models.py | 98 ---------- monitoring/ciwatch/ciwatch/populate.py | 45 ----- monitoring/ciwatch/ciwatch/static/hover.js | 23 --- monitoring/ciwatch/ciwatch/static/style.css | 65 ------- monitoring/ciwatch/ciwatch/static/verified.js | 33 ---- .../ciwatch/templates/_contact.html.jinja | 1 - .../ciwatch/templates/_header.html.jinja | 3 - .../ciwatch/templates/_usage.html.jinja | 7 - .../ciwatch/templates/index.html.jinja | 111 ----------- .../ciwatch/templates/project.html.jinja | 109 ----------- monitoring/ciwatch/ciwatch/views.py | 38 ---- monitoring/ciwatch/requirements.txt | 4 - monitoring/ciwatch/run.py | 18 -- monitoring/ciwatch/setup.py | 41 ---- 26 files changed, 1363 deletions(-) delete mode 100644 monitoring/ciwatch/LICENSE delete mode 100644 monitoring/ciwatch/README.md delete mode 100644 monitoring/ciwatch/ci-watch.conf.sample delete mode 100644 monitoring/ciwatch/ciwatch.wsgi delete mode 100644 monitoring/ciwatch/ciwatch/__init__.py delete mode 100644 monitoring/ciwatch/ciwatch/api.py delete mode 100644 monitoring/ciwatch/ciwatch/cache.py delete mode 100644 monitoring/ciwatch/ciwatch/config.py delete mode 100644 monitoring/ciwatch/ciwatch/db.py delete mode 100644 monitoring/ciwatch/ciwatch/events.py delete mode 100644 monitoring/ciwatch/ciwatch/filters.py delete mode 100644 monitoring/ciwatch/ciwatch/log.py delete mode 100644 monitoring/ciwatch/ciwatch/models.py delete mode 100644 monitoring/ciwatch/ciwatch/populate.py delete mode 100644 monitoring/ciwatch/ciwatch/static/hover.js delete mode 100644 monitoring/ciwatch/ciwatch/static/style.css delete mode 100644 monitoring/ciwatch/ciwatch/static/verified.js delete mode 100644 monitoring/ciwatch/ciwatch/templates/_contact.html.jinja delete mode 100644 monitoring/ciwatch/ciwatch/templates/_header.html.jinja delete mode 100644 monitoring/ciwatch/ciwatch/templates/_usage.html.jinja delete mode 100644 monitoring/ciwatch/ciwatch/templates/index.html.jinja delete mode 100644 monitoring/ciwatch/ciwatch/templates/project.html.jinja delete mode 100644 monitoring/ciwatch/ciwatch/views.py delete mode 100644 monitoring/ciwatch/requirements.txt delete mode 100644 monitoring/ciwatch/run.py delete mode 100644 monitoring/ciwatch/setup.py diff --git a/monitoring/ciwatch/LICENSE b/monitoring/ciwatch/LICENSE deleted file mode 100644 index 68c771a..0000000 --- a/monitoring/ciwatch/LICENSE +++ /dev/null @@ -1,176 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - diff --git a/monitoring/ciwatch/README.md b/monitoring/ciwatch/README.md deleted file mode 100644 index cb9f9e6..0000000 --- a/monitoring/ciwatch/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# CI Watch - -## Configuration - -Configuration is stored in the `ci-watch.conf` file. Importantly, you can -specify a directory to store the `third-party-ci.log` file (data\_dir) as well -as the database to connect to. Look at `ci-watch.conf.sample` for an example. - -Other settings should be self explanatory based on the provided configuration -file. - -## Installation - -From this folder, run the following commands. - -``` -pip install -r requirements.txt -# Note that this step requires the `ci-watch.conf` file. -pip install -e . -``` - -These instructions are for development and testing installations. - -## Usage - -At the moment, this package provides three commands. - -`ci-watch-server`. -Launch a development server. - -`ci-watch-stream-events`. -Stream events from Gerrit and append valid events to `third-party-ci.log`. - -`ci-watch-populate-database`. -Add all entries from `third-party-ci.log` to the database. - -## State of the project - -This project is a work in progress and the code is pretty rough in some places. - -## TODO - -* Add tests. -* Use a different cache other than SimpleCache. It is not threadsafe. We - should use something like redis instead. - -These items are far from the only work needed for this project. - - -## Acknowledgements - -This code was originally forked from John Griffith's sos-ci project. Some of it -can still be found in the code and configuration file. diff --git a/monitoring/ciwatch/ci-watch.conf.sample b/monitoring/ciwatch/ci-watch.conf.sample deleted file mode 100644 index 5b94e88..0000000 --- a/monitoring/ciwatch/ci-watch.conf.sample +++ /dev/null @@ -1,18 +0,0 @@ -[AccountInfo] -gerrit_ssh_key = /path/to/private/key -gerrit_username = your_username -gerrit_host = review.openstack.org -gerrit_port = 29418 - -[Data] -debug = True -data_dir = /var/data - -[database] -connection = sqlite:///:memory - -[misc] -; This is a more complete list that is not missing any CI's from the wiki -; It may include projects that don't actually do the third party CI thing -; projects = cinder,nova,swift,rally,murano,keystone,ironic,octavia,os-brick,neutron,tempest,neutron-lbaas,devstack,designate,manila -projects = cinder,nova,swift,rally,murano,ironic,octavia,os-brick,neutron,neutron-lbaas,devstack,manila diff --git a/monitoring/ciwatch/ciwatch.wsgi b/monitoring/ciwatch/ciwatch.wsgi deleted file mode 100644 index 7d9ae44..0000000 --- a/monitoring/ciwatch/ciwatch.wsgi +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sys -sys.path.insert(0, "/var/www/ciwatch") -from ciwatch import app as application diff --git a/monitoring/ciwatch/ciwatch/__init__.py b/monitoring/ciwatch/ciwatch/__init__.py deleted file mode 100644 index 58ac9fb..0000000 --- a/monitoring/ciwatch/ciwatch/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from flask import Flask - -app = Flask(__name__) - -from ciwatch import views # noqa -from ciwatch import filters # noqa - - -__version__ = "0.0.1" - - -def main(): - app.run(debug=True, host='0.0.0.0') - - -if __name__ == '__main__': - main() diff --git a/monitoring/ciwatch/ciwatch/api.py b/monitoring/ciwatch/ciwatch/api.py deleted file mode 100644 index 64fdf62..0000000 --- a/monitoring/ciwatch/ciwatch/api.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from collections import OrderedDict -from datetime import datetime, timedelta - -from flask import request -from sqlalchemy import and_ -from sqlalchemy import desc - -from ciwatch import db -from ciwatch.models import CiServer, Project, PatchSet - - -TIME_OPTIONS = OrderedDict([ # Map time options to hours - ("24 hours", 24), - ("48 hours", 48), - ("7 days", 7 * 24), -]) - -DEFAULT_TIME_OPTION = "24 hours" -DEFAULT_PROJECT = "cinder" - - -def _get_ci_info_for_patch_sets(ci, patch_sets): - ci_info = {"name": ci.name, "trusted": ci.trusted, "results": []} - for patch_set in patch_sets: - for comment in patch_set.comments: - if comment.ci_server_id == ci.id: - ci_info["results"].append(comment) - break - else: # nobreak - ci_info["results"].append(None) - return ci_info - - -def get_projects(): - return db.session.query(Project).order_by(Project.name).all() - - -def get_ci_servers(): - return db.session.query(CiServer).order_by( - desc(CiServer.trusted), CiServer.name).all() - - -def get_patch_sets(project, since): - return db.session.query(PatchSet).filter( - and_(PatchSet.project == project, PatchSet.created >= since) - ).order_by(PatchSet.created.desc()).all() - - -def get_time_options(): - return TIME_OPTIONS.keys() - - -def get_context(): - project = request.args.get('project', DEFAULT_PROJECT) - time = request.args.get('time', DEFAULT_TIME_OPTION) - since = datetime.now() - timedelta(hours=TIME_OPTIONS[time]) - project = db.session.query(Project).filter( - Project.name == project).one() - patch_sets = get_patch_sets(project=project, since=since) - results = OrderedDict() - for ci in get_ci_servers(): - ci_info = _get_ci_info_for_patch_sets(ci, patch_sets) - if any(result for result in ci_info["results"]): - results[ci.ci_owner] = results.get(ci.ci_owner, []) - results[ci.ci_owner].append( - _get_ci_info_for_patch_sets(ci, patch_sets)) - - return {"time_options": get_time_options(), - "time_option": time, - "patch_sets": patch_sets, - "project": project, - "projects": get_projects(), - "user_results": results} diff --git a/monitoring/ciwatch/ciwatch/cache.py b/monitoring/ciwatch/ciwatch/cache.py deleted file mode 100644 index 61526df..0000000 --- a/monitoring/ciwatch/ciwatch/cache.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from functools import wraps - -from flask import request -from werkzeug.contrib.cache import SimpleCache - - -cache = SimpleCache() - - -def cached(func): - @wraps(func) - def wrapper(*args, **kwargs): - key = _get_cache_key() - result = cache.get(key) - if result is None: - result = func(*args, **kwargs) - cache.set(key, result, timeout=60) - return result - return wrapper - - -def _get_cache_key(): - args = request.args - return request.path + str([(key, args[key]) for key in sorted(args)]) diff --git a/monitoring/ciwatch/ciwatch/config.py b/monitoring/ciwatch/ciwatch/config.py deleted file mode 100644 index d71b284..0000000 --- a/monitoring/ciwatch/ciwatch/config.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - -from iniparse import INIConfig - - -def get_config(): - this_file = os.path.dirname(os.path.realpath(__file__)) - this_dir = os.path.dirname(this_file) - conf_files = [os.path.join(this_dir, 'ci-watch.conf'), - '/etc/ciwatch/ci-watch.conf'] - # Read first existing conf file, ignore the rest - for conf_file in conf_files: - if os.path.exists(conf_file): - return INIConfig(open(conf_file)) - raise Exception('Could not read configuration from %s' % conf_files) - -cfg = get_config() - - -def get_projects(): - projects = [] - for name in cfg.misc.projects.split(','): - projects.append(name) - return projects diff --git a/monitoring/ciwatch/ciwatch/db.py b/monitoring/ciwatch/ciwatch/db.py deleted file mode 100644 index f9c0b56..0000000 --- a/monitoring/ciwatch/ciwatch/db.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -from ciwatch import models -from ciwatch.config import cfg, get_projects - - -engine = create_engine(cfg.database.connection) -Session = sessionmaker() -Session.configure(bind=engine) -models.Base.metadata.create_all(engine) -session = Session() - - -def create_projects(): - for name in get_projects(): - get_or_create(models.Project, - commit_=False, - name=name) - session.commit() - - -def update_or_create_comment(commit_=True, **kwargs): - comment = session.query(models.Comment).filter_by( - ci_server_id=kwargs['ci_server_id'], - patch_set_id=kwargs['patch_set_id']).scalar() - if comment is not None: - for key, value in kwargs.iteritems(): - setattr(comment, key, value) - else: - session.add(models.Comment(**kwargs)) - if commit_: - session.commit() - - -def get_or_create(model, commit_=True, **kwargs): - result = session.query(model).filter_by(**kwargs).first() - if not result: - result = model(**kwargs) - session.add(result) - if commit_: - session.commit() - return result diff --git a/monitoring/ciwatch/ciwatch/events.py b/monitoring/ciwatch/ciwatch/events.py deleted file mode 100644 index 712e65c..0000000 --- a/monitoring/ciwatch/ciwatch/events.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import paramiko -import time -from datetime import datetime -import re - -from ciwatch import db, models -from ciwatch.log import logger, DATA_DIR -from ciwatch.config import cfg, get_projects - - -def _process_project_name(project_name): - return project_name.split('/')[-1] - - -def _process_event(event): - comment = event['comment'] - # Find all the CIs voting in this comment - lines = comment.splitlines() - event['ci-status'] = {} - for line in lines: - possible_results = "FAILURE|SUCCESS|NOT_REGISTERED|UNSTABLE" - pattern = re.compile("[-*]\s+([^\s*]+)\s+(http[^\s*]+) : (%s)" % - possible_results) - match = pattern.search(line) - if match is not None: - ci_name = match.group(1) - log_url = match.group(2) - result = match.group(3) - event['ci-status'][ci_name] = { - "result": result, - "log_url": log_url} - - -def _is_ci_user(name): - return 'CI' in name or 'Jenkins' in name - - -# Check if this is a third party CI event -def _is_valid(event): - if (event.get('type', 'nill') == 'comment-added' and - _is_ci_user(event['author'].get('name', '')) and - _process_project_name(event['change']['project']) in get_projects() and - event['change']['branch'] == 'master'): - return True - return False - - -def _store_event(event): - with open(DATA_DIR + '/third-party-ci.log', 'a') as f: - json.dump(event, f) - f.write('\n') - add_event_to_db(event) - return event - - -class GerritEventStream(object): - def __init__(self): - - logger.debug('Connecting to %(host)s:%(port)d as ' - '%(user)s using %(key)s', - {'user': cfg.AccountInfo.gerrit_username, - 'key': cfg.AccountInfo.gerrit_ssh_key, - 'host': cfg.AccountInfo.gerrit_host, - 'port': int(cfg.AccountInfo.gerrit_port)}) - - self.ssh = paramiko.SSHClient() - self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - connected = False - while not connected: - try: - self.ssh.connect(cfg.AccountInfo.gerrit_host, - int(cfg.AccountInfo.gerrit_port), - cfg.AccountInfo.gerrit_username, - key_filename=cfg.AccountInfo.gerrit_ssh_key) - connected = True - except paramiko.SSHException as e: - logger.error('%s', e) - logger.warn('Gerrit may be down, will pause and retry...') - time.sleep(10) - - self.stdin, self.stdout, self.stderr =\ - self.ssh.exec_command("gerrit stream-events") - - def __iter__(self): - return self - - def next(self): - return self.stdout.readline() - - -def parse_json_event(event): - try: - event = json.loads(event) - except Exception as ex: - logger.error('Failed json.loads on event: %s', event) - logger.exception(ex) - return None - if _is_valid(event): - _process_event(event) - logger.info('Parsed valid event: %s', event) - return event - return None - - -def add_event_to_db(event, commit_=True): - project = db.session.query(models.Project).filter( - models.Project.name == _process_project_name( - event["change"]["project"])).one() - patch_set = db.get_or_create( - models.PatchSet, - commit_=False, - project_id=project.id, - ref=event['patchSet']['ref'], - commit_message=event['change']['commitMessage'], - created=datetime.fromtimestamp( - int(event['patchSet']['createdOn']))) - - owner_name = event["author"]["name"] - owner = db.get_or_create(models.CiOwner, name=owner_name) - trusted = (event["author"]["username"] == "jenkins") - - if trusted and "approvals" in event: - if event["approvals"][0]["value"] in ("+1", "+2"): - patch_set.verified = True - elif event["approvals"][0]["value"] in ("-1", "-2"): - patch_set.verified = False - - for ci, data in event['ci-status'].iteritems(): - ci_server = db.get_or_create(models.CiServer, - commit_=False, - name=ci, - trusted=trusted, - ci_owner_id=owner.id) - db.update_or_create_comment(commit_=False, - result=data["result"], - log_url=data["log_url"], - ci_server_id=ci_server.id, - patch_set_id=patch_set.id) - if commit_: - db.session.commit() - - -def main(): - db.create_projects() # This will make sure the database has projects in it - while True: - try: - events = GerritEventStream() - except paramiko.SSHException as ex: - logger.exception('Error connecting to Gerrit: %s', ex) - time.sleep(60) - for event in events: - event = parse_json_event(event) - if event is not None: - _store_event(event) - - -if __name__ == '__main__': - main() diff --git a/monitoring/ciwatch/ciwatch/filters.py b/monitoring/ciwatch/ciwatch/filters.py deleted file mode 100644 index 088e895..0000000 --- a/monitoring/ciwatch/ciwatch/filters.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import re - -from jinja2 import evalcontextfilter, Markup, escape - -from ciwatch import app - - -_paragraph_re = re.compile(r'(?:\r\n|\r|\n){2,}') - - -@app.template_filter() -@evalcontextfilter -def nl2br(eval_ctx, value): - result = u'\n\n'.join(u'

%s

' % p.replace('\n', '
\n') - for p in _paragraph_re.split(escape(value))) - if eval_ctx.autoescape: - result = Markup(result) - return result diff --git a/monitoring/ciwatch/ciwatch/log.py b/monitoring/ciwatch/ciwatch/log.py deleted file mode 100644 index 1f887df..0000000 --- a/monitoring/ciwatch/ciwatch/log.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -from logging import handlers -import os - -from ciwatch.config import cfg - - -def setup_logger(name): - - logger = logging.getLogger(name) - logger.setLevel(logging.DEBUG) - log_formatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s]" - " %(message)s") - - file_handler =\ - handlers.RotatingFileHandler(name, - maxBytes=1048576, - backupCount=2,) - logger.addHandler(file_handler) - - console_handler = logging.StreamHandler() - console_handler.setFormatter(log_formatter) - logger.addHandler(console_handler) - return logger - - -DATA_DIR =\ - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/data' -if cfg.Data.data_dir: - DATA_DIR = cfg.Data.data_dir - -logger = setup_logger(DATA_DIR + '/ci-watch.log') diff --git a/monitoring/ciwatch/ciwatch/models.py b/monitoring/ciwatch/ciwatch/models.py deleted file mode 100644 index 4787a81..0000000 --- a/monitoring/ciwatch/ciwatch/models.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import Boolean, Column, DateTime, Integer, String, ForeignKey -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship, backref - - -Base = declarative_base() - - -class Project(Base): - __tablename__ = "projects" - - id = Column(Integer, primary_key=True) - name = Column(String(128), unique=True) - - def __repr__(self): - return "" % self.name - - -class PatchSet(Base): - __tablename__ = "patch_sets" - - id = Column(Integer, primary_key=True) - created = Column(DateTime) - # ref = Column(String(64), unique=True) - ref = Column(String(64)) # Why are there duplicate refs? - # Verified only represents Jenkin's vote - verified = Column(Boolean, nullable=True, default=None) - - commit_message = Column(String(4096)) - - project_id = Column(Integer, ForeignKey('projects.id')) - project = relationship("Project", backref=backref('patch_sets', - order_by=id)) - - def __repr__(self): - return "" % ( - self.created, self.ref) - - -class Comment(Base): - __tablename__ = "comments" - - id = Column(Integer, primary_key=True) - result = Column(String(64)) - log_url = Column(String(1024), nullable=True, default=None) - - ci_server_id = Column(Integer, ForeignKey('ci_servers.id')) - ci_server = relationship("CiServer", backref=backref('comments', - order_by=id)) - - patch_set_id = Column(Integer, ForeignKey('patch_sets.id')) - patch_set = relationship("PatchSet", backref=backref('comments', - order_by=id)) - - def __repr__(self): - return "" % ( - self.log_url, self.result) - - -class CiServer(Base): - __tablename__ = "ci_servers" - - id = Column(Integer, primary_key=True) - name = Column(String(128)) - - # Official OpenStack CIs are trusted (e.g., Jenkins) - trusted = Column(Boolean, default=False) - - ci_owner_id = Column(Integer, ForeignKey('ci_owners.id')) - ci_owner = relationship('CiOwner', backref=backref('ci_servers', - order_by=id)) - - def __repr__(self): - return "" % self.name - - -class CiOwner(Base): - __tablename__ = "ci_owners" - - id = Column(Integer, primary_key=True) - name = Column(String(128), unique=True) - - def __repr__(self): - return "" % self.name diff --git a/monitoring/ciwatch/ciwatch/populate.py b/monitoring/ciwatch/ciwatch/populate.py deleted file mode 100644 index 71be64d..0000000 --- a/monitoring/ciwatch/ciwatch/populate.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - -from ciwatch import db -from ciwatch.events import parse_json_event, add_event_to_db -from ciwatch.log import DATA_DIR - - -def get_data(): - data = [] - with open(os.path.join(DATA_DIR, 'third-party-ci.log')) as file_: - for line in file_: - event = parse_json_event(line) - if event is not None: - data.append(event) - return data - - -def load_data(): - data = get_data() - for event in data: - add_event_to_db(event, commit_=False) - db.session.commit() - - -def main(): - db.create_projects() - load_data() - - -if __name__ == '__main__': - main() diff --git a/monitoring/ciwatch/ciwatch/static/hover.js b/monitoring/ciwatch/ciwatch/static/hover.js deleted file mode 100644 index cbedd77..0000000 --- a/monitoring/ciwatch/ciwatch/static/hover.js +++ /dev/null @@ -1,23 +0,0 @@ -// Code adapted from https://css-tricks.com/row-and-column-highlighting/ -$(document).ready(function () { - $("table").delegate("td.result","mouseover mouseleave", function(e) { - if (e.type == "mouseover") { - $(this).parent().addClass("hover"); - $("colgroup").eq($(this).index()).addClass("hover"); - } - else { - $(this).parent().removeClass("hover"); - $("colgroup").eq($(this).index()).removeClass("hover"); - } - }); - $("table").delegate("td.ci-name","mouseover mouseleave", function(e) { - if (e.type == "mouseover") { - $(this).parent().addClass("hover"); - } - else { - $(this).parent().removeClass("hover"); - } - }); - - $('[data-toggle="popover"]').popover({trigger: 'hover'}); -}); diff --git a/monitoring/ciwatch/ciwatch/static/style.css b/monitoring/ciwatch/ciwatch/static/style.css deleted file mode 100644 index 42a9461..0000000 --- a/monitoring/ciwatch/ciwatch/static/style.css +++ /dev/null @@ -1,65 +0,0 @@ -table.table { - white-space: nowrap; -} - -.hover { - background-color: #EEE; -} - -.failure { - color: #C00; - font-weight: bold; -} - -.success { - color: #0C0; - font-weight: bold; -} - -.unstable { - color: #BB0; - font-weight: bold; -} - -.unregistered { - color: #333; - font-weight: bold; -} - -.ci-user { - background-color: #DDD !important; - font-style: italic; -} - -.blank-row { - height: 20px; -} - -.popover { - max-width: 100%; -} - -.no-style-link:link { - text-decoration: none; - color: #333; -} - -.no-style-link:visited { - text-decoration: none; - color: #333; -} - -.no-style-link:hover { - text-decoration: none; - color: #333; -} - -.no-style-link:active { - text-decoration: none; - color: #333; -} - -.glyphicon-none:before { - content: "\2122"; - color: transparent !important; -} diff --git a/monitoring/ciwatch/ciwatch/static/verified.js b/monitoring/ciwatch/ciwatch/static/verified.js deleted file mode 100644 index ee3eaf9..0000000 --- a/monitoring/ciwatch/ciwatch/static/verified.js +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) 2015 Tintri. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -var toggle1 = function () { - $(".verified-1").css("background-color", "#FAA"); - $(this).one("click", toggle2); -} - -var toggle2 = function () { - $(".verified-1").css("background-color", ""); - $(this).one("click", toggle1); -} - -$(document).ready(function () { - $("colgroup").each(function (i, elem) { - if ($(elem).hasClass("verified-1")) { - $("#results").find("td").filter(":nth-child(" + (i + 1) + ")").addClass("verified-1"); - } - }); - $("#verified-1-button").one("click", toggle1); -}); diff --git a/monitoring/ciwatch/ciwatch/templates/_contact.html.jinja b/monitoring/ciwatch/ciwatch/templates/_contact.html.jinja deleted file mode 100644 index e3c477c..0000000 --- a/monitoring/ciwatch/ciwatch/templates/_contact.html.jinja +++ /dev/null @@ -1 +0,0 @@ -

Send feedback to openstack-dev.tintri.com

diff --git a/monitoring/ciwatch/ciwatch/templates/_header.html.jinja b/monitoring/ciwatch/ciwatch/templates/_header.html.jinja deleted file mode 100644 index e40d27d..0000000 --- a/monitoring/ciwatch/ciwatch/templates/_header.html.jinja +++ /dev/null @@ -1,3 +0,0 @@ -
-

CI Watch — an OpenStack third-party CI monitoring tool

-
diff --git a/monitoring/ciwatch/ciwatch/templates/_usage.html.jinja b/monitoring/ciwatch/ciwatch/templates/_usage.html.jinja deleted file mode 100644 index ea95e55..0000000 --- a/monitoring/ciwatch/ciwatch/templates/_usage.html.jinja +++ /dev/null @@ -1,7 +0,0 @@ -

- Each project has a table of recent CI results. - Columns plot patch sets and rows plot CI servers. - Each cell shows results for the row's CI server and the column's patch set. - Click the icons to view relevant logs. - Newest results are shown furthest to the left. -

diff --git a/monitoring/ciwatch/ciwatch/templates/index.html.jinja b/monitoring/ciwatch/ciwatch/templates/index.html.jinja deleted file mode 100644 index 3b7a137..0000000 --- a/monitoring/ciwatch/ciwatch/templates/index.html.jinja +++ /dev/null @@ -1,111 +0,0 @@ - - - - - - - - - - - -
- - {% include '_header.html.jinja' %} - -
-

Select a project to view CI results

-
-
- {% for project in projects %} -
- -

{{ project.name }}

- -
- {% endfor %} -
-
- -

Usage

- {% include '_usage.html.jinja' %} - -
-
Legend
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
IconMeaning
- - - - Link to patch set on Gerrit
-

- - -

- 		CI voted SUCCESS for patch set

-

- - -

- 		CI voted FAILURE for patch set
-

- - -

- 		CI voted UNSTABLE for patch set
-

N

- 		CI voted NOT_REGISTERED patch set
- - - CI has not yet voted on this patch set
-
- -

Contact Us

- {% include '_contact.html.jinja' %} - -
- - - - - - diff --git a/monitoring/ciwatch/ciwatch/templates/project.html.jinja b/monitoring/ciwatch/ciwatch/templates/project.html.jinja deleted file mode 100644 index 911b6c2..0000000 --- a/monitoring/ciwatch/ciwatch/templates/project.html.jinja +++ /dev/null @@ -1,109 +0,0 @@ - - - - - - - - - - -
- {% include '_header.html.jinja' %} - {% include '_usage.html.jinja' %} - {% include '_contact.html.jinja' %} -
Viewing results for {{ project.name }} from the past {{ request.args.get('time', time_option) }}.
-
- -
-
- - -
- -
- -
    - {% for option in time_options %} -
  • {{ option }}
    • - {% endfor %} -
-
- - -
- -
- -
- - - - {% for patch_set in patch_sets %} - {% if patch_set.verified is not none and not patch_set.verfied %} - - {% else %} - - {% endif %} - {% endfor %} - - - - - - {% for patch_set in patch_sets %} - - {% endfor %} - - - {% for owner, results in user_results.iteritems() %} - - {% for ci in results %} - - - {% for comment in ci["results"] %} - - {% endfor %} - - {% endfor %} - - {% endfor %} - -
Patch Set - - - -
{{ owner.name }}
{{ ci["name"] }} - {% if comment is not none %} - - {% if comment.result == "SUCCESS" %} -

- {% elif comment.result == "FAILURE" %} -

- {% elif comment.result == "UNSTABLE" %} -

- {% elif comment.result == "NOT_REGISTERED" %} -

N

- {% endif %} - - {% endif %} -
-
- - - - - - - - diff --git a/monitoring/ciwatch/ciwatch/views.py b/monitoring/ciwatch/ciwatch/views.py deleted file mode 100644 index 31a5430..0000000 --- a/monitoring/ciwatch/ciwatch/views.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from flask import render_template -from sqlalchemy.orm.exc import NoResultFound -from werkzeug.exceptions import abort - -from ciwatch import app -from ciwatch.api import get_context -from ciwatch.api import get_projects -from ciwatch.cache import cached - - -@app.route("/") -@app.route("/index") -@app.route("/home") -def home(): - return render_template('index.html.jinja', projects=get_projects()) - - -@app.route("/project") -@cached -def project(): - try: - return render_template("project.html.jinja", **get_context()) - except NoResultFound: - abort(404) diff --git a/monitoring/ciwatch/requirements.txt b/monitoring/ciwatch/requirements.txt deleted file mode 100644 index 157ff9a..0000000 --- a/monitoring/ciwatch/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -flask -sqlalchemy -iniparse -paramiko diff --git a/monitoring/ciwatch/run.py b/monitoring/ciwatch/run.py deleted file mode 100644 index de80b25..0000000 --- a/monitoring/ciwatch/run.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import ciwatch - -if __name__ == "__main__": - ciwatch.app.run(debug=True, host='0.0.0.0') diff --git a/monitoring/ciwatch/setup.py b/monitoring/ciwatch/setup.py deleted file mode 100644 index 84ae731..0000000 --- a/monitoring/ciwatch/setup.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2015 Tintri. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from os.path import join, dirname - -from setuptools import setup - -import ciwatch - - -setup( - name='ci-watch', - version=ciwatch.__version__, - long_description=open(join(dirname(__file__), 'README.md')).read(), - entry_points={ - 'console_scripts': [ - 'ci-watch-server = ciwatch:main', - 'ci-watch-populate-database = ciwatch.populate:main', - 'ci-watch-stream-events = ciwatch.events:main', - ], - }, - install_requires=[ - "flask", - "sqlalchemy", - "iniparse", - "paramiko", - ] -)