diff --git a/ciwatch/config.py b/ciwatch/config.py index d71b284..8dd9273 100644 --- a/ciwatch/config.py +++ b/ciwatch/config.py @@ -17,22 +17,31 @@ import os from iniparse import INIConfig -def get_config(): - this_file = os.path.dirname(os.path.realpath(__file__)) - this_dir = os.path.dirname(this_file) - conf_files = [os.path.join(this_dir, 'ci-watch.conf'), - '/etc/ciwatch/ci-watch.conf'] - # Read first existing conf file, ignore the rest - for conf_file in conf_files: - if os.path.exists(conf_file): - return INIConfig(open(conf_file)) - raise Exception('Could not read configuration from %s' % conf_files) +class Config(object): -cfg = get_config() + def __init__(self): + self.cfg = self.get_config() + if self.cfg.Data.data_dir: + self.DATA_DIR = self.cfg.Data.data_dir + else: + self.DATA_DIR = os.path.dirname( + os.path.dirname(os.path.abspath(__file__))) + '/data' + def get_config(self): + this_file = os.path.dirname(os.path.realpath(__file__)) + this_dir = os.path.dirname(this_file) + conf_files = [os.path.join(this_dir, 'ci-watch.conf'), + '/etc/ciwatch/ci-watch.conf'] + # Read first existing conf file, ignore the rest + for conf_file in conf_files: + if os.path.exists(conf_file): + return INIConfig(open(conf_file)) + else: + raise Exception( + 'Could not read configuration from %s' % conf_files) -def get_projects(): - projects = [] - for name in cfg.misc.projects.split(','): - projects.append(name) - return projects + def get_projects(self): + projects = [] + for name in self.cfg.misc.projects.split(','): + projects.append(name) + return projects diff --git a/ciwatch/db.py b/ciwatch/db.py index 419fec3..4d548f2 100644 --- a/ciwatch/db.py +++ b/ciwatch/db.py @@ -15,12 +15,12 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from ciwatch.config import cfg -from ciwatch.config import get_projects +from ciwatch.config import Config from ciwatch import models -engine = create_engine(cfg.database.connection) +config = Config() +engine = create_engine(config.cfg.database.connection) Session = sessionmaker() Session.configure(bind=engine) models.Base.metadata.create_all(engine) @@ -28,7 +28,7 @@ session = Session() def create_projects(): - for name in get_projects(): + for name in config.get_projects(): get_or_create(models.Project, commit_=False, name=name) diff --git a/ciwatch/events.py b/ciwatch/events.py index bda6e85..2b5145e 100644 --- a/ciwatch/events.py +++ b/ciwatch/events.py @@ -18,14 +18,10 @@ import paramiko import re import time +from ciwatch.config import Config from ciwatch import db -from ciwatch import models - -from ciwatch.log import DATA_DIR from ciwatch.log import logger - -from ciwatch.config import cfg -from ciwatch.config import get_projects +from ciwatch import models def _process_project_name(project_name): @@ -56,18 +52,18 @@ def _is_ci_user(name): # Check if this is a third party CI event -def _is_valid(event): +def _is_valid(event, projects): if (event.get('type', 'nill') == 'comment-added' and _is_ci_user(event['author'].get('name', '')) and _process_project_name( - event['change']['project']) in get_projects() and + event['change']['project']) in projects and event['change']['branch'] == 'master'): return True return False -def _store_event(event): - with open(DATA_DIR + '/third-party-ci.log', 'a') as f: +def _store_event(event, datadir): + with open(datadir + '/third-party-ci.log', 'a') as f: json.dump(event, f) f.write('\n') add_event_to_db(event) @@ -75,7 +71,7 @@ def _store_event(event): class GerritEventStream(object): - def __init__(self): + def __init__(self, cfg): logger.debug('Connecting to %(host)s:%(port)d as ' '%(user)s using %(key)s', @@ -110,14 +106,14 @@ class GerritEventStream(object): return self.stdout.readline() -def parse_json_event(event): +def parse_json_event(event, projects): try: event = json.loads(event) except Exception as ex: logger.error('Failed json.loads on event: %s', event) logger.exception(ex) return None - if _is_valid(event): + if _is_valid(event, projects): _process_event(event) logger.info('Parsed valid event: %s', event) return event @@ -163,17 +159,18 @@ def add_event_to_db(event, commit_=True): def main(): + config = Config() db.create_projects() # This will make sure the database has projects in it while True: try: - events = GerritEventStream() + events = GerritEventStream(config.cfg) except paramiko.SSHException as ex: logger.exception('Error connecting to Gerrit: %s', ex) time.sleep(60) for event in events: - event = parse_json_event(event) + event = parse_json_event(event, config.get_projects()) if event is not None: - _store_event(event) + _store_event(event, config.DATA_DIR) if __name__ == '__main__': diff --git a/ciwatch/log.py b/ciwatch/log.py index 1f887df..b32a16f 100644 --- a/ciwatch/log.py +++ b/ciwatch/log.py @@ -14,9 +14,9 @@ import logging from logging import handlers -import os -from ciwatch.config import cfg +from ciwatch.config import Config +config = Config() def setup_logger(name): @@ -38,9 +38,4 @@ def setup_logger(name): return logger -DATA_DIR =\ - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/data' -if cfg.Data.data_dir: - DATA_DIR = cfg.Data.data_dir - -logger = setup_logger(DATA_DIR + '/ci-watch.log') +logger = setup_logger(config.DATA_DIR + '/ci-watch.log') diff --git a/ciwatch/populate.py b/ciwatch/populate.py index db20fa3..8a077bb 100644 --- a/ciwatch/populate.py +++ b/ciwatch/populate.py @@ -14,32 +14,35 @@ import os +from ciwatch.config import Config from ciwatch import db from ciwatch.events import add_event_to_db from ciwatch.events import parse_json_event -from ciwatch.log import DATA_DIR -def get_data(): +def get_data(datafile, projects): data = [] - with open(os.path.join(DATA_DIR, 'third-party-ci.log')) as file_: + with open(datafile) as file_: for line in file_: - event = parse_json_event(line) + event = parse_json_event(line, projects) if event is not None: data.append(event) return data -def load_data(): - data = get_data() +def load_data(data): for event in data: add_event_to_db(event, commit_=False) db.session.commit() def main(): + config = Config() + projects = config.get_projects() + datafile = os.path.join(config.DATA_DIR, 'third-party-ci.log') db.create_projects() - load_data() + data = get_data(datafile, projects) + load_data(data) if __name__ == '__main__':