Refactor configuration handling
This change should make it a bit easier to see when configuration is read in, and would allow to further centralize parsing and loading of the conf file. We should not 'import config' each time when we need to get access to the settings. Instead, the settings should be loaded once, and then explicitly passed around. This patch does not make that happen yet, but is the first step. Change-Id: If9ad1222ceb806337a745845ee6e76605bab5fd3
This commit is contained in:
parent
32315f3e13
commit
52315eedd7
|
@ -17,22 +17,31 @@ import os
|
|||
from iniparse import INIConfig
|
||||
|
||||
|
||||
def get_config():
|
||||
this_file = os.path.dirname(os.path.realpath(__file__))
|
||||
this_dir = os.path.dirname(this_file)
|
||||
conf_files = [os.path.join(this_dir, 'ci-watch.conf'),
|
||||
'/etc/ciwatch/ci-watch.conf']
|
||||
# Read first existing conf file, ignore the rest
|
||||
for conf_file in conf_files:
|
||||
if os.path.exists(conf_file):
|
||||
return INIConfig(open(conf_file))
|
||||
raise Exception('Could not read configuration from %s' % conf_files)
|
||||
class Config(object):
|
||||
|
||||
cfg = get_config()
|
||||
def __init__(self):
|
||||
self.cfg = self.get_config()
|
||||
if self.cfg.Data.data_dir:
|
||||
self.DATA_DIR = self.cfg.Data.data_dir
|
||||
else:
|
||||
self.DATA_DIR = os.path.dirname(
|
||||
os.path.dirname(os.path.abspath(__file__))) + '/data'
|
||||
|
||||
def get_config(self):
|
||||
this_file = os.path.dirname(os.path.realpath(__file__))
|
||||
this_dir = os.path.dirname(this_file)
|
||||
conf_files = [os.path.join(this_dir, 'ci-watch.conf'),
|
||||
'/etc/ciwatch/ci-watch.conf']
|
||||
# Read first existing conf file, ignore the rest
|
||||
for conf_file in conf_files:
|
||||
if os.path.exists(conf_file):
|
||||
return INIConfig(open(conf_file))
|
||||
else:
|
||||
raise Exception(
|
||||
'Could not read configuration from %s' % conf_files)
|
||||
|
||||
def get_projects():
|
||||
projects = []
|
||||
for name in cfg.misc.projects.split(','):
|
||||
projects.append(name)
|
||||
return projects
|
||||
def get_projects(self):
|
||||
projects = []
|
||||
for name in self.cfg.misc.projects.split(','):
|
||||
projects.append(name)
|
||||
return projects
|
||||
|
|
|
@ -15,12 +15,12 @@
|
|||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from ciwatch.config import cfg
|
||||
from ciwatch.config import get_projects
|
||||
from ciwatch.config import Config
|
||||
from ciwatch import models
|
||||
|
||||
|
||||
engine = create_engine(cfg.database.connection)
|
||||
config = Config()
|
||||
engine = create_engine(config.cfg.database.connection)
|
||||
Session = sessionmaker()
|
||||
Session.configure(bind=engine)
|
||||
models.Base.metadata.create_all(engine)
|
||||
|
@ -28,7 +28,7 @@ session = Session()
|
|||
|
||||
|
||||
def create_projects():
|
||||
for name in get_projects():
|
||||
for name in config.get_projects():
|
||||
get_or_create(models.Project,
|
||||
commit_=False,
|
||||
name=name)
|
||||
|
|
|
@ -18,14 +18,10 @@ import paramiko
|
|||
import re
|
||||
import time
|
||||
|
||||
from ciwatch.config import Config
|
||||
from ciwatch import db
|
||||
from ciwatch import models
|
||||
|
||||
from ciwatch.log import DATA_DIR
|
||||
from ciwatch.log import logger
|
||||
|
||||
from ciwatch.config import cfg
|
||||
from ciwatch.config import get_projects
|
||||
from ciwatch import models
|
||||
|
||||
|
||||
def _process_project_name(project_name):
|
||||
|
@ -56,18 +52,18 @@ def _is_ci_user(name):
|
|||
|
||||
|
||||
# Check if this is a third party CI event
|
||||
def _is_valid(event):
|
||||
def _is_valid(event, projects):
|
||||
if (event.get('type', 'nill') == 'comment-added' and
|
||||
_is_ci_user(event['author'].get('name', '')) and
|
||||
_process_project_name(
|
||||
event['change']['project']) in get_projects() and
|
||||
event['change']['project']) in projects and
|
||||
event['change']['branch'] == 'master'):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _store_event(event):
|
||||
with open(DATA_DIR + '/third-party-ci.log', 'a') as f:
|
||||
def _store_event(event, datadir):
|
||||
with open(datadir + '/third-party-ci.log', 'a') as f:
|
||||
json.dump(event, f)
|
||||
f.write('\n')
|
||||
add_event_to_db(event)
|
||||
|
@ -75,7 +71,7 @@ def _store_event(event):
|
|||
|
||||
|
||||
class GerritEventStream(object):
|
||||
def __init__(self):
|
||||
def __init__(self, cfg):
|
||||
|
||||
logger.debug('Connecting to %(host)s:%(port)d as '
|
||||
'%(user)s using %(key)s',
|
||||
|
@ -110,14 +106,14 @@ class GerritEventStream(object):
|
|||
return self.stdout.readline()
|
||||
|
||||
|
||||
def parse_json_event(event):
|
||||
def parse_json_event(event, projects):
|
||||
try:
|
||||
event = json.loads(event)
|
||||
except Exception as ex:
|
||||
logger.error('Failed json.loads on event: %s', event)
|
||||
logger.exception(ex)
|
||||
return None
|
||||
if _is_valid(event):
|
||||
if _is_valid(event, projects):
|
||||
_process_event(event)
|
||||
logger.info('Parsed valid event: %s', event)
|
||||
return event
|
||||
|
@ -163,17 +159,18 @@ def add_event_to_db(event, commit_=True):
|
|||
|
||||
|
||||
def main():
|
||||
config = Config()
|
||||
db.create_projects() # This will make sure the database has projects in it
|
||||
while True:
|
||||
try:
|
||||
events = GerritEventStream()
|
||||
events = GerritEventStream(config.cfg)
|
||||
except paramiko.SSHException as ex:
|
||||
logger.exception('Error connecting to Gerrit: %s', ex)
|
||||
time.sleep(60)
|
||||
for event in events:
|
||||
event = parse_json_event(event)
|
||||
event = parse_json_event(event, config.get_projects())
|
||||
if event is not None:
|
||||
_store_event(event)
|
||||
_store_event(event, config.DATA_DIR)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
@ -14,9 +14,9 @@
|
|||
|
||||
import logging
|
||||
from logging import handlers
|
||||
import os
|
||||
|
||||
from ciwatch.config import cfg
|
||||
from ciwatch.config import Config
|
||||
config = Config()
|
||||
|
||||
|
||||
def setup_logger(name):
|
||||
|
@ -38,9 +38,4 @@ def setup_logger(name):
|
|||
return logger
|
||||
|
||||
|
||||
DATA_DIR =\
|
||||
os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/data'
|
||||
if cfg.Data.data_dir:
|
||||
DATA_DIR = cfg.Data.data_dir
|
||||
|
||||
logger = setup_logger(DATA_DIR + '/ci-watch.log')
|
||||
logger = setup_logger(config.DATA_DIR + '/ci-watch.log')
|
||||
|
|
|
@ -14,32 +14,35 @@
|
|||
|
||||
import os
|
||||
|
||||
from ciwatch.config import Config
|
||||
from ciwatch import db
|
||||
from ciwatch.events import add_event_to_db
|
||||
from ciwatch.events import parse_json_event
|
||||
from ciwatch.log import DATA_DIR
|
||||
|
||||
|
||||
def get_data():
|
||||
def get_data(datafile, projects):
|
||||
data = []
|
||||
with open(os.path.join(DATA_DIR, 'third-party-ci.log')) as file_:
|
||||
with open(datafile) as file_:
|
||||
for line in file_:
|
||||
event = parse_json_event(line)
|
||||
event = parse_json_event(line, projects)
|
||||
if event is not None:
|
||||
data.append(event)
|
||||
return data
|
||||
|
||||
|
||||
def load_data():
|
||||
data = get_data()
|
||||
def load_data(data):
|
||||
for event in data:
|
||||
add_event_to_db(event, commit_=False)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def main():
|
||||
config = Config()
|
||||
projects = config.get_projects()
|
||||
datafile = os.path.join(config.DATA_DIR, 'third-party-ci.log')
|
||||
db.create_projects()
|
||||
load_data()
|
||||
data = get_data(datafile, projects)
|
||||
load_data(data)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
Loading…
Reference in New Issue