Refactor configuration handling
This change should make it a bit easier to see when configuration is read in, and would allow to further centralize parsing and loading of the conf file. We should not 'import config' each time when we need to get access to the settings. Instead, the settings should be loaded once, and then explicitly passed around. This patch does not make that happen yet, but is the first step. Change-Id: If9ad1222ceb806337a745845ee6e76605bab5fd3
This commit is contained in:
parent
32315f3e13
commit
52315eedd7
|
@ -17,7 +17,17 @@ import os
|
||||||
from iniparse import INIConfig
|
from iniparse import INIConfig
|
||||||
|
|
||||||
|
|
||||||
def get_config():
|
class Config(object):
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.cfg = self.get_config()
|
||||||
|
if self.cfg.Data.data_dir:
|
||||||
|
self.DATA_DIR = self.cfg.Data.data_dir
|
||||||
|
else:
|
||||||
|
self.DATA_DIR = os.path.dirname(
|
||||||
|
os.path.dirname(os.path.abspath(__file__))) + '/data'
|
||||||
|
|
||||||
|
def get_config(self):
|
||||||
this_file = os.path.dirname(os.path.realpath(__file__))
|
this_file = os.path.dirname(os.path.realpath(__file__))
|
||||||
this_dir = os.path.dirname(this_file)
|
this_dir = os.path.dirname(this_file)
|
||||||
conf_files = [os.path.join(this_dir, 'ci-watch.conf'),
|
conf_files = [os.path.join(this_dir, 'ci-watch.conf'),
|
||||||
|
@ -26,13 +36,12 @@ def get_config():
|
||||||
for conf_file in conf_files:
|
for conf_file in conf_files:
|
||||||
if os.path.exists(conf_file):
|
if os.path.exists(conf_file):
|
||||||
return INIConfig(open(conf_file))
|
return INIConfig(open(conf_file))
|
||||||
raise Exception('Could not read configuration from %s' % conf_files)
|
else:
|
||||||
|
raise Exception(
|
||||||
|
'Could not read configuration from %s' % conf_files)
|
||||||
|
|
||||||
cfg = get_config()
|
def get_projects(self):
|
||||||
|
|
||||||
|
|
||||||
def get_projects():
|
|
||||||
projects = []
|
projects = []
|
||||||
for name in cfg.misc.projects.split(','):
|
for name in self.cfg.misc.projects.split(','):
|
||||||
projects.append(name)
|
projects.append(name)
|
||||||
return projects
|
return projects
|
||||||
|
|
|
@ -15,12 +15,12 @@
|
||||||
from sqlalchemy import create_engine
|
from sqlalchemy import create_engine
|
||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
from ciwatch.config import cfg
|
from ciwatch.config import Config
|
||||||
from ciwatch.config import get_projects
|
|
||||||
from ciwatch import models
|
from ciwatch import models
|
||||||
|
|
||||||
|
|
||||||
engine = create_engine(cfg.database.connection)
|
config = Config()
|
||||||
|
engine = create_engine(config.cfg.database.connection)
|
||||||
Session = sessionmaker()
|
Session = sessionmaker()
|
||||||
Session.configure(bind=engine)
|
Session.configure(bind=engine)
|
||||||
models.Base.metadata.create_all(engine)
|
models.Base.metadata.create_all(engine)
|
||||||
|
@ -28,7 +28,7 @@ session = Session()
|
||||||
|
|
||||||
|
|
||||||
def create_projects():
|
def create_projects():
|
||||||
for name in get_projects():
|
for name in config.get_projects():
|
||||||
get_or_create(models.Project,
|
get_or_create(models.Project,
|
||||||
commit_=False,
|
commit_=False,
|
||||||
name=name)
|
name=name)
|
||||||
|
|
|
@ -18,14 +18,10 @@ import paramiko
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
from ciwatch.config import Config
|
||||||
from ciwatch import db
|
from ciwatch import db
|
||||||
from ciwatch import models
|
|
||||||
|
|
||||||
from ciwatch.log import DATA_DIR
|
|
||||||
from ciwatch.log import logger
|
from ciwatch.log import logger
|
||||||
|
from ciwatch import models
|
||||||
from ciwatch.config import cfg
|
|
||||||
from ciwatch.config import get_projects
|
|
||||||
|
|
||||||
|
|
||||||
def _process_project_name(project_name):
|
def _process_project_name(project_name):
|
||||||
|
@ -56,18 +52,18 @@ def _is_ci_user(name):
|
||||||
|
|
||||||
|
|
||||||
# Check if this is a third party CI event
|
# Check if this is a third party CI event
|
||||||
def _is_valid(event):
|
def _is_valid(event, projects):
|
||||||
if (event.get('type', 'nill') == 'comment-added' and
|
if (event.get('type', 'nill') == 'comment-added' and
|
||||||
_is_ci_user(event['author'].get('name', '')) and
|
_is_ci_user(event['author'].get('name', '')) and
|
||||||
_process_project_name(
|
_process_project_name(
|
||||||
event['change']['project']) in get_projects() and
|
event['change']['project']) in projects and
|
||||||
event['change']['branch'] == 'master'):
|
event['change']['branch'] == 'master'):
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _store_event(event):
|
def _store_event(event, datadir):
|
||||||
with open(DATA_DIR + '/third-party-ci.log', 'a') as f:
|
with open(datadir + '/third-party-ci.log', 'a') as f:
|
||||||
json.dump(event, f)
|
json.dump(event, f)
|
||||||
f.write('\n')
|
f.write('\n')
|
||||||
add_event_to_db(event)
|
add_event_to_db(event)
|
||||||
|
@ -75,7 +71,7 @@ def _store_event(event):
|
||||||
|
|
||||||
|
|
||||||
class GerritEventStream(object):
|
class GerritEventStream(object):
|
||||||
def __init__(self):
|
def __init__(self, cfg):
|
||||||
|
|
||||||
logger.debug('Connecting to %(host)s:%(port)d as '
|
logger.debug('Connecting to %(host)s:%(port)d as '
|
||||||
'%(user)s using %(key)s',
|
'%(user)s using %(key)s',
|
||||||
|
@ -110,14 +106,14 @@ class GerritEventStream(object):
|
||||||
return self.stdout.readline()
|
return self.stdout.readline()
|
||||||
|
|
||||||
|
|
||||||
def parse_json_event(event):
|
def parse_json_event(event, projects):
|
||||||
try:
|
try:
|
||||||
event = json.loads(event)
|
event = json.loads(event)
|
||||||
except Exception as ex:
|
except Exception as ex:
|
||||||
logger.error('Failed json.loads on event: %s', event)
|
logger.error('Failed json.loads on event: %s', event)
|
||||||
logger.exception(ex)
|
logger.exception(ex)
|
||||||
return None
|
return None
|
||||||
if _is_valid(event):
|
if _is_valid(event, projects):
|
||||||
_process_event(event)
|
_process_event(event)
|
||||||
logger.info('Parsed valid event: %s', event)
|
logger.info('Parsed valid event: %s', event)
|
||||||
return event
|
return event
|
||||||
|
@ -163,17 +159,18 @@ def add_event_to_db(event, commit_=True):
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
config = Config()
|
||||||
db.create_projects() # This will make sure the database has projects in it
|
db.create_projects() # This will make sure the database has projects in it
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
events = GerritEventStream()
|
events = GerritEventStream(config.cfg)
|
||||||
except paramiko.SSHException as ex:
|
except paramiko.SSHException as ex:
|
||||||
logger.exception('Error connecting to Gerrit: %s', ex)
|
logger.exception('Error connecting to Gerrit: %s', ex)
|
||||||
time.sleep(60)
|
time.sleep(60)
|
||||||
for event in events:
|
for event in events:
|
||||||
event = parse_json_event(event)
|
event = parse_json_event(event, config.get_projects())
|
||||||
if event is not None:
|
if event is not None:
|
||||||
_store_event(event)
|
_store_event(event, config.DATA_DIR)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -14,9 +14,9 @@
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from logging import handlers
|
from logging import handlers
|
||||||
import os
|
|
||||||
|
|
||||||
from ciwatch.config import cfg
|
from ciwatch.config import Config
|
||||||
|
config = Config()
|
||||||
|
|
||||||
|
|
||||||
def setup_logger(name):
|
def setup_logger(name):
|
||||||
|
@ -38,9 +38,4 @@ def setup_logger(name):
|
||||||
return logger
|
return logger
|
||||||
|
|
||||||
|
|
||||||
DATA_DIR =\
|
logger = setup_logger(config.DATA_DIR + '/ci-watch.log')
|
||||||
os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/data'
|
|
||||||
if cfg.Data.data_dir:
|
|
||||||
DATA_DIR = cfg.Data.data_dir
|
|
||||||
|
|
||||||
logger = setup_logger(DATA_DIR + '/ci-watch.log')
|
|
||||||
|
|
|
@ -14,32 +14,35 @@
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
from ciwatch.config import Config
|
||||||
from ciwatch import db
|
from ciwatch import db
|
||||||
from ciwatch.events import add_event_to_db
|
from ciwatch.events import add_event_to_db
|
||||||
from ciwatch.events import parse_json_event
|
from ciwatch.events import parse_json_event
|
||||||
from ciwatch.log import DATA_DIR
|
|
||||||
|
|
||||||
|
|
||||||
def get_data():
|
def get_data(datafile, projects):
|
||||||
data = []
|
data = []
|
||||||
with open(os.path.join(DATA_DIR, 'third-party-ci.log')) as file_:
|
with open(datafile) as file_:
|
||||||
for line in file_:
|
for line in file_:
|
||||||
event = parse_json_event(line)
|
event = parse_json_event(line, projects)
|
||||||
if event is not None:
|
if event is not None:
|
||||||
data.append(event)
|
data.append(event)
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
def load_data():
|
def load_data(data):
|
||||||
data = get_data()
|
|
||||||
for event in data:
|
for event in data:
|
||||||
add_event_to_db(event, commit_=False)
|
add_event_to_db(event, commit_=False)
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
config = Config()
|
||||||
|
projects = config.get_projects()
|
||||||
|
datafile = os.path.join(config.DATA_DIR, 'third-party-ci.log')
|
||||||
db.create_projects()
|
db.create_projects()
|
||||||
load_data()
|
data = get_data(datafile, projects)
|
||||||
|
load_data(data)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
Loading…
Reference in New Issue