Required SQL reporters
On the way towards a fully scale out scheduler we need to move the times database from the local filesystem into the SQL database. Therefore we need to make at least one SQL connection mandatory. SQL reporters are required (an implied sql reporter is added to every pipeline, explicit sql reporters ignored) Change-Id: I30723f9b320b9f2937cc1d7ff3267519161bc380 Depends-On: https://review.opendev.org/621479 Story: 2007192 Task: 38329
This commit is contained in:
parent
9e0c733aac
commit
9ab527971f
|
@ -28,9 +28,7 @@ name=opendev
|
|||
driver=git
|
||||
baseurl=https://opendev.org
|
||||
|
||||
[connection "mysql"]
|
||||
name=mysql
|
||||
driver=sql
|
||||
[database]
|
||||
dburi=mysql+pymysql://zuul:%(ZUUL_MYSQL_PASSWORD)s@mysql/zuul
|
||||
|
||||
[web]
|
||||
|
|
|
@ -17,11 +17,9 @@
|
|||
success:
|
||||
gerrit:
|
||||
Verified: 1
|
||||
mysql:
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -1
|
||||
mysql:
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
|
@ -48,8 +46,6 @@
|
|||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
mysql:
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
mysql:
|
||||
|
|
|
@ -148,6 +148,9 @@ service in Zuul, and a connection to Gerrit.
|
|||
user=zuul
|
||||
sshkey=/home/zuul/.ssh/id_rsa
|
||||
|
||||
[database]
|
||||
dburi=mysql+pymysql://zuul:secret@mysql/zuul
|
||||
|
||||
See :ref:`components` and :ref:`connections` for more details.
|
||||
|
||||
The following tells Zuul to read its configuration from and operate on
|
||||
|
|
|
@ -5,6 +5,7 @@ Admin Reference
|
|||
:maxdepth: 3
|
||||
|
||||
tenants
|
||||
database
|
||||
connections
|
||||
drivers/index
|
||||
client
|
||||
|
|
|
@ -0,0 +1,49 @@
|
|||
:title: Database Configuration
|
||||
|
||||
.. _database:
|
||||
|
||||
Database Configuration
|
||||
======================
|
||||
|
||||
The database configuration is located in the ``database`` section of
|
||||
``zuul.conf``:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[database]
|
||||
dburi=<URI>
|
||||
|
||||
The following options can be defined in this section.
|
||||
|
||||
.. attr:: database
|
||||
|
||||
.. attr:: dburi
|
||||
:required:
|
||||
|
||||
Database connection information in the form of a URI understood
|
||||
by SQLAlchemy. See `The SQLAlchemy manual
|
||||
<https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>`_
|
||||
for more information.
|
||||
|
||||
The driver will automatically set up the database creating and managing
|
||||
the necessary tables. Therefore the provided user should have sufficient
|
||||
permissions to manage the database. For example:
|
||||
|
||||
.. code-block:: sql
|
||||
|
||||
GRANT ALL ON my_database TO 'my_user'@'%';
|
||||
|
||||
.. attr:: pool_recycle
|
||||
:default: 1
|
||||
|
||||
Tune the pool_recycle value. See `The SQLAlchemy manual on pooling
|
||||
<http://docs.sqlalchemy.org/en/latest/core/pooling.html#setting-pool-recycle>`_
|
||||
for more information.
|
||||
|
||||
.. attr:: table_prefix
|
||||
:default: ''
|
||||
|
||||
The string to prefix the table names. This makes it possible to run
|
||||
several zuul deployments against the same database. This can be useful
|
||||
if you rely on external databases which are not under your control.
|
||||
The default is to have no prefix.
|
|
@ -5,6 +5,10 @@
|
|||
SQL
|
||||
===
|
||||
|
||||
.. warning::
|
||||
|
||||
This driver is deprecated, use :attr:`database` configuration instead.
|
||||
|
||||
The SQL driver supports reporters only. Only one connection per
|
||||
database is permitted.
|
||||
|
||||
|
|
|
@ -0,0 +1,12 @@
|
|||
---
|
||||
upgrade:
|
||||
- |
|
||||
The components zuul-scheduler and zuul-web now require database
|
||||
configuration. There is now only one supported database connection. If
|
||||
multiple sql connections have been configured only the first one will be
|
||||
used.
|
||||
|
||||
deprecations:
|
||||
- |
|
||||
Defining database connections using :attr:`<sql connection>` is now
|
||||
deprecated. Refer to :ref:`database` how to configure the database now.
|
|
@ -66,6 +66,7 @@ from git.exc import NoSuchPathError
|
|||
import yaml
|
||||
import paramiko
|
||||
|
||||
from zuul.driver.sql.sqlconnection import DatabaseSession
|
||||
from zuul.model import Change
|
||||
from zuul.rpcclient import RPCClient
|
||||
|
||||
|
@ -74,7 +75,7 @@ from zuul.driver.git import GitDriver
|
|||
from zuul.driver.smtp import SMTPDriver
|
||||
from zuul.driver.github import GithubDriver
|
||||
from zuul.driver.timer import TimerDriver
|
||||
from zuul.driver.sql import SQLDriver
|
||||
from zuul.driver.sql import SQLDriver, sqlconnection
|
||||
from zuul.driver.bubblewrap import BubblewrapDriver
|
||||
from zuul.driver.nullwrap import NullwrapDriver
|
||||
from zuul.driver.mqtt import MQTTDriver
|
||||
|
@ -311,12 +312,19 @@ class GitlabDriverMock(GitlabDriver):
|
|||
return connection
|
||||
|
||||
|
||||
class SQLDriverMock(SQLDriver):
|
||||
|
||||
def getConnection(self, name, config):
|
||||
return FakeSqlConnection(self, name, config)
|
||||
|
||||
|
||||
class TestConnectionRegistry(ConnectionRegistry):
|
||||
def __init__(self, changes: Dict[str, Dict[str, Change]],
|
||||
config: ConfigParser, additional_event_queues,
|
||||
upstream_root: str, rpcclient: RPCClient, poller_events,
|
||||
git_url_with_auth: bool,
|
||||
add_cleanup: Callable[[Callable[[], None]], None]):
|
||||
add_cleanup: Callable[[Callable[[], None]], None],
|
||||
fake_sql: bool):
|
||||
self.connections = OrderedDict()
|
||||
self.drivers = {}
|
||||
|
||||
|
@ -330,7 +338,10 @@ class TestConnectionRegistry(ConnectionRegistry):
|
|||
rpcclient, git_url_with_auth))
|
||||
self.registerDriver(SMTPDriver())
|
||||
self.registerDriver(TimerDriver())
|
||||
self.registerDriver(SQLDriver())
|
||||
if fake_sql:
|
||||
self.registerDriver(SQLDriverMock())
|
||||
else:
|
||||
self.registerDriver(SQLDriver())
|
||||
self.registerDriver(BubblewrapDriver())
|
||||
self.registerDriver(NullwrapDriver())
|
||||
self.registerDriver(MQTTDriver())
|
||||
|
@ -2889,6 +2900,37 @@ class FakeBuild(object):
|
|||
return repos
|
||||
|
||||
|
||||
class FakeZuulDatabaseSession(DatabaseSession):
|
||||
|
||||
def __exit__(self, etype, value, tb):
|
||||
pass
|
||||
|
||||
|
||||
def FakeOrmSessionFactory():
|
||||
class FakeBackendSession:
|
||||
def add(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
return FakeBackendSession()
|
||||
|
||||
|
||||
class FakeSqlConnection(sqlconnection.SQLConnection):
|
||||
|
||||
def __init__(self, driver, connection_name, connection_config):
|
||||
connection_config['dburi'] = 'postgresql://example.com'
|
||||
super().__init__(driver, connection_name, connection_config)
|
||||
self.session = FakeOrmSessionFactory
|
||||
|
||||
def onLoad(self):
|
||||
pass
|
||||
|
||||
def onStop(self):
|
||||
pass
|
||||
|
||||
def getSession(self):
|
||||
return FakeZuulDatabaseSession(self)
|
||||
|
||||
|
||||
class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
|
||||
result = None
|
||||
|
||||
|
@ -3587,12 +3629,12 @@ class ZuulWebFixture(fixtures.Fixture):
|
|||
additional_event_queues, upstream_root: str,
|
||||
rpcclient: RPCClient, poller_events, git_url_with_auth: bool,
|
||||
add_cleanup: Callable[[Callable[[], None]], None],
|
||||
test_root, info=None, zk_hosts=None):
|
||||
test_root, info=None, zk_hosts=None, fake_sql=True):
|
||||
super(ZuulWebFixture, self).__init__()
|
||||
self.gearman_server_port = gearman_server_port
|
||||
self.connections = TestConnectionRegistry(
|
||||
changes, config, additional_event_queues, upstream_root, rpcclient,
|
||||
poller_events, git_url_with_auth, add_cleanup)
|
||||
poller_events, git_url_with_auth, add_cleanup, fake_sql)
|
||||
self.connections.configure(
|
||||
config,
|
||||
include_drivers=[zuul.driver.sql.SQLDriver,
|
||||
|
@ -3847,6 +3889,7 @@ class SchedulerTestApp:
|
|||
additional_event_queues, upstream_root: str,
|
||||
rpcclient: RPCClient, poller_events, git_url_with_auth: bool,
|
||||
source_only: bool,
|
||||
fake_sql: bool,
|
||||
add_cleanup: Callable[[Callable[[], None]], None]):
|
||||
|
||||
self.log = log
|
||||
|
@ -3868,7 +3911,7 @@ class SchedulerTestApp:
|
|||
self.connections = TestConnectionRegistry(
|
||||
self.changes, self.config, additional_event_queues,
|
||||
upstream_root, rpcclient, poller_events,
|
||||
git_url_with_auth, add_cleanup)
|
||||
git_url_with_auth, add_cleanup, fake_sql)
|
||||
self.connections.configure(self.config, source_only=source_only)
|
||||
|
||||
self.sched.registerConnections(self.connections)
|
||||
|
@ -3917,12 +3960,13 @@ class SchedulerTestManager:
|
|||
changes: Dict[str, Dict[str, Change]], additional_event_queues,
|
||||
upstream_root: str, rpcclient: RPCClient, poller_events,
|
||||
git_url_with_auth: bool, source_only: bool,
|
||||
fake_sql: bool,
|
||||
add_cleanup: Callable[[Callable[[], None]], None])\
|
||||
-> SchedulerTestApp:
|
||||
app = SchedulerTestApp(log, config, zk_config, changes,
|
||||
additional_event_queues, upstream_root,
|
||||
rpcclient, poller_events, git_url_with_auth,
|
||||
source_only, add_cleanup)
|
||||
source_only, fake_sql, add_cleanup)
|
||||
self.instances.append(app)
|
||||
return app
|
||||
|
||||
|
@ -4030,6 +4074,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
git_url_with_auth: bool = False
|
||||
log_console_port: int = 19885
|
||||
source_only: bool = False
|
||||
fake_sql: bool = True
|
||||
|
||||
def __getattr__(self, name):
|
||||
"""Allows to access fake connections the old way, e.g., using
|
||||
|
@ -4161,7 +4206,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
executor_connections = TestConnectionRegistry(
|
||||
self.changes, self.config, self.additional_event_queues,
|
||||
self.upstream_root, self.rpcclient, self.poller_events,
|
||||
self.git_url_with_auth, self.addCleanup)
|
||||
self.git_url_with_auth, self.addCleanup, True)
|
||||
executor_connections.configure(self.config,
|
||||
source_only=self.source_only)
|
||||
self.executor_server = RecordingExecutorServer(
|
||||
|
@ -4180,7 +4225,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
self.log, self.config, self.zk_config, self.changes,
|
||||
self.additional_event_queues, self.upstream_root, self.rpcclient,
|
||||
self.poller_events, self.git_url_with_auth, self.source_only,
|
||||
self.addCleanup)
|
||||
self.fake_sql, self.addCleanup)
|
||||
|
||||
if hasattr(self, 'fake_github'):
|
||||
self.additional_event_queues.append(
|
||||
|
@ -4271,6 +4316,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
# Make test_root persist after ansible run for .flag test
|
||||
config.set('executor', 'trusted_rw_paths', self.test_root)
|
||||
self.setupAllProjectKeys(config)
|
||||
|
||||
return config
|
||||
|
||||
def setupSimpleLayout(self, config: ConfigParser):
|
||||
|
@ -5273,7 +5319,21 @@ class SSLZuulTestCase(ZuulTestCase):
|
|||
|
||||
|
||||
class ZuulDBTestCase(ZuulTestCase):
|
||||
fake_sql = False
|
||||
|
||||
def setup_config(self, config_file: str):
|
||||
def _setup_fixture(config, section_name):
|
||||
if (config.get(section_name, 'dburi') ==
|
||||
'$MYSQL_FIXTURE_DBURI$'):
|
||||
f = MySQLSchemaFixture()
|
||||
self.useFixture(f)
|
||||
config.set(section_name, 'dburi', f.dburi)
|
||||
elif (config.get(section_name, 'dburi') ==
|
||||
'$POSTGRESQL_FIXTURE_DBURI$'):
|
||||
f = PostgresqlSchemaFixture()
|
||||
self.useFixture(f)
|
||||
config.set(section_name, 'dburi', f.dburi)
|
||||
|
||||
config = super(ZuulDBTestCase, self).setup_config(config_file)
|
||||
for section_name in config.sections():
|
||||
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
|
||||
|
@ -5282,16 +5342,11 @@ class ZuulDBTestCase(ZuulTestCase):
|
|||
continue
|
||||
|
||||
if config.get(section_name, 'driver') == 'sql':
|
||||
if (config.get(section_name, 'dburi') ==
|
||||
'$MYSQL_FIXTURE_DBURI$'):
|
||||
f = MySQLSchemaFixture()
|
||||
self.useFixture(f)
|
||||
config.set(section_name, 'dburi', f.dburi)
|
||||
elif (config.get(section_name, 'dburi') ==
|
||||
'$POSTGRESQL_FIXTURE_DBURI$'):
|
||||
f = PostgresqlSchemaFixture()
|
||||
self.useFixture(f)
|
||||
config.set(section_name, 'dburi', f.dburi)
|
||||
_setup_fixture(config, section_name)
|
||||
|
||||
if 'database' in config.sections():
|
||||
_setup_fixture(config, 'database')
|
||||
|
||||
return config
|
||||
|
||||
|
||||
|
|
|
@ -7,15 +7,10 @@
|
|||
success:
|
||||
gerrit:
|
||||
Verified: 1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_mysql_failures: null
|
||||
resultsdb_postgresql: null
|
||||
resultsdb_postgresql_failures: null
|
||||
resultsdb_failures: null
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
|
@ -30,15 +25,10 @@
|
|||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
resultsdb_mysql: null
|
||||
resultsdb_mysql_failures: null
|
||||
resultsdb_postgresql: null
|
||||
resultsdb_postgresql_failures: null
|
||||
resultsdb_failures: null
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
|
@ -51,14 +41,8 @@
|
|||
gerrit:
|
||||
- event: ref-updated
|
||||
ref: ^refs/tags/.*$
|
||||
success:
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
failure:
|
||||
resultsdb_mysql: null
|
||||
resultsdb_mysql_failures: null
|
||||
resultsdb_postgresql: null
|
||||
resultsdb_postgresql_failures: null
|
||||
resultsdb_failures: null
|
||||
|
||||
- job:
|
||||
name: base
|
||||
|
|
|
@ -7,13 +7,9 @@
|
|||
success:
|
||||
gerrit:
|
||||
Verified: 1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
|
||||
- job:
|
||||
name: base
|
||||
|
|
|
@ -7,13 +7,9 @@
|
|||
success:
|
||||
gerrit:
|
||||
Verified: 1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
|
|
|
@ -7,13 +7,9 @@
|
|||
success:
|
||||
gerrit:
|
||||
Verified: 1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -1
|
||||
resultsdb_mysql: null
|
||||
resultsdb_postgresql: null
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
|
|
|
@ -5,11 +5,9 @@
|
|||
gerrit:
|
||||
- event: patchset-created
|
||||
success:
|
||||
resultsdb_mysql: null
|
||||
gerrit:
|
||||
Verified: 1
|
||||
failure:
|
||||
resultsdb_mysql: null
|
||||
gerrit:
|
||||
Verified: -1
|
||||
|
||||
|
|
|
@ -29,3 +29,7 @@ realm=zuul.example.com
|
|||
client_id=zuul.example.com
|
||||
issuer_id=zuul_operator
|
||||
secret=NoDanaOnlyZuul
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -43,3 +43,7 @@ realm=myOIDC2
|
|||
client_id=zuul
|
||||
issuer_id=http://oidc2
|
||||
scope=openid profile email special-scope
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -30,3 +30,7 @@ client_id=zuul.example.com
|
|||
issuer_id=zuul_operator
|
||||
secret=NoDanaOnlyZuul
|
||||
max_validity_time=5
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -29,3 +29,7 @@ realm=zuul.example.com
|
|||
client_id=zuul.example.com
|
||||
issuer_id=zuul_operator
|
||||
secret=NoDanaOnlyZuul
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -25,3 +25,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -29,3 +29,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -24,3 +24,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -21,10 +21,10 @@ server=review.example.com
|
|||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[connection resultsdb]
|
||||
driver=sql
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
||||
[connection smtp]
|
||||
driver=smtp
|
||||
server=localhost
|
||||
|
|
|
@ -30,3 +30,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -30,3 +30,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -37,3 +37,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -34,3 +34,7 @@ default_to=you@example.com
|
|||
|
||||
[web]
|
||||
static_cache_expiry=1200
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -25,3 +25,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -23,3 +23,6 @@ driver=elasticsearch
|
|||
uri=localhost:9200
|
||||
use_ssl=true
|
||||
verify_certs=false
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
|
|
@ -46,3 +46,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -30,3 +30,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -33,3 +33,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -31,3 +31,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -33,3 +33,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -29,3 +29,7 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -18,3 +18,7 @@ webhook_token=0000000000000000000000000000000000000000
|
|||
app_id=1
|
||||
app_key=$APP_KEY_FIXTURE$
|
||||
server=github.enterprise.io
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -26,3 +26,7 @@ sshkey=/home/zuul/.ssh/id_rsa
|
|||
driver=github
|
||||
sshkey=/home/zuul/.ssh/id_rsa
|
||||
server=github.enterprise.io
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -16,3 +16,7 @@ git_dir=/tmp/zuul-test/executor-git
|
|||
driver=gitlab
|
||||
server=gitlab
|
||||
api_token=0000000000000000000000000000000000000000
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -36,3 +36,7 @@ default_to=you@example.com
|
|||
[web]
|
||||
static_cache_expiry=1200
|
||||
root=https://zuul.example.com/
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -21,5 +21,8 @@ sshkey=fake_id_rsa1
|
|||
[connection mqtt]
|
||||
driver=mqtt
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[web]
|
||||
root=https://tenant.example.com/
|
||||
|
|
|
@ -17,3 +17,7 @@ driver=pagure
|
|||
server=pagure
|
||||
api_token=0000000000000000000000000000000000000000
|
||||
source_whitelist=::ffff:127.0.0.1
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -16,3 +16,7 @@ git_dir=/tmp/zuul-test/executor-git
|
|||
driver=pagure
|
||||
server=pagure
|
||||
api_token=0000000000000000000000000000000000000000
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -20,3 +20,7 @@ webhook_token=00000000000000000000000000000000000000000
|
|||
driver=gerrit
|
||||
server=review.example.com
|
||||
user=jenkins
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
|
|
|
@ -18,8 +18,7 @@ server=review.example.com
|
|||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[connection resultsdb_mysql]
|
||||
driver=sql
|
||||
[database]
|
||||
dburi=mysql+pymysql://bad:creds@host/db
|
||||
|
||||
[connection resultsdb_mysql_failures]
|
||||
|
|
|
@ -0,0 +1,26 @@
|
|||
[gearman]
|
||||
server=127.0.0.1
|
||||
|
||||
[scheduler]
|
||||
tenant_config=main.yaml
|
||||
|
||||
[merger]
|
||||
git_dir=/tmp/zuul-test/merger-git
|
||||
git_user_email=zuul@example.com
|
||||
git_user_name=zuul
|
||||
|
||||
[executor]
|
||||
git_dir=/tmp/zuul-test/executor-git
|
||||
|
||||
[connection gerrit]
|
||||
driver=gerrit
|
||||
server=review.example.com
|
||||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[connection resultsdb_failures]
|
||||
driver=sql
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
|
@ -0,0 +1,26 @@
|
|||
[gearman]
|
||||
server=127.0.0.1
|
||||
|
||||
[scheduler]
|
||||
tenant_config=main.yaml
|
||||
|
||||
[merger]
|
||||
git_dir=/tmp/zuul-test/merger-git
|
||||
git_user_email=zuul@example.com
|
||||
git_user_name=zuul
|
||||
|
||||
[executor]
|
||||
git_dir=/tmp/zuul-test/executor-git
|
||||
|
||||
[connection gerrit]
|
||||
driver=gerrit
|
||||
server=review.example.com
|
||||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[database]
|
||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
||||
|
||||
[connection resultsdb_failures]
|
||||
driver=sql
|
||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
|
@ -18,20 +18,10 @@ server=review.example.com
|
|||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[connection resultsdb_mysql]
|
||||
driver=sql
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
table_prefix=prefix_
|
||||
|
||||
[connection resultsdb_mysql_failures]
|
||||
[connection resultsdb_failures]
|
||||
driver=sql
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[connection resultsdb_postgresql]
|
||||
driver=sql
|
||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
||||
table_prefix=prefix_
|
||||
|
||||
[connection resultsdb_postgresql_failures]
|
||||
driver=sql
|
||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
|
@ -18,18 +18,11 @@ server=review.example.com
|
|||
user=jenkins
|
||||
sshkey=fake_id_rsa1
|
||||
|
||||
[connection resultsdb_mysql]
|
||||
driver=sql
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[connection resultsdb_mysql_failures]
|
||||
driver=sql
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[connection resultsdb_postgresql]
|
||||
[database]
|
||||
driver=sql
|
||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
||||
table_prefix=prefix_
|
||||
|
||||
[connection resultsdb_postgresql_failures]
|
||||
[connection resultsdb_failures]
|
||||
driver=sql
|
||||
dburi=$POSTGRESQL_FIXTURE_DBURI$
|
|
@ -35,3 +35,6 @@ server=localhost
|
|||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
|
|
@ -31,6 +31,9 @@ port=25
|
|||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
||||
|
||||
[database]
|
||||
dburi=$MYSQL_FIXTURE_DBURI$
|
||||
|
||||
[web]
|
||||
static_cache_expiry=1200
|
||||
root=https://zuul.example.com/
|
||||
|
|
|
@ -11,13 +11,18 @@
|
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import configparser
|
||||
import os
|
||||
import re
|
||||
import textwrap
|
||||
import time
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from tests.base import ZuulTestCase, ZuulDBTestCase, AnsibleZuulTestCase
|
||||
import zuul
|
||||
from tests.base import ZuulTestCase, FIXTURE_DIR, \
|
||||
PostgresqlSchemaFixture, MySQLSchemaFixture, ZuulDBTestCase, \
|
||||
BaseTestCase, AnsibleZuulTestCase
|
||||
|
||||
|
||||
def _get_reporter_from_connection_name(reporters, connection_name):
|
||||
|
@ -60,8 +65,8 @@ class TestConnections(ZuulTestCase):
|
|||
'civoter')
|
||||
|
||||
|
||||
class TestSQLConnection(ZuulDBTestCase):
|
||||
config_file = 'zuul-sql-driver.conf'
|
||||
class TestSQLConnectionMysql(ZuulDBTestCase):
|
||||
config_file = 'zuul-sql-driver-mysql.conf'
|
||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||
expected_table_prefix = ''
|
||||
|
||||
|
@ -80,8 +85,7 @@ class TestSQLConnection(ZuulDBTestCase):
|
|||
|
||||
def test_sql_tables_created(self):
|
||||
"Test the tables for storing results are created properly"
|
||||
self._sql_tables_created('resultsdb_mysql')
|
||||
self._sql_tables_created('resultsdb_postgresql')
|
||||
self._sql_tables_created('database')
|
||||
|
||||
def _sql_indexes_created(self, connection_name):
|
||||
connection = self.scheds.first.connections.connections[connection_name]
|
||||
|
@ -115,8 +119,7 @@ class TestSQLConnection(ZuulDBTestCase):
|
|||
|
||||
def test_sql_indexes_created(self):
|
||||
"Test the indexes are created properly"
|
||||
self._sql_indexes_created('resultsdb_mysql')
|
||||
self._sql_indexes_created('resultsdb_postgresql')
|
||||
self._sql_indexes_created('database')
|
||||
|
||||
def test_sql_results(self):
|
||||
"Test results are entered into an sql table"
|
||||
|
@ -224,8 +227,7 @@ class TestSQLConnection(ZuulDBTestCase):
|
|||
self.orderedRelease()
|
||||
self.waitUntilSettled()
|
||||
|
||||
check_results('resultsdb_mysql')
|
||||
check_results('resultsdb_postgresql')
|
||||
check_results('database')
|
||||
|
||||
def test_sql_results_retry_builds(self):
|
||||
"Test that retry results are entered into an sql table correctly"
|
||||
|
@ -305,8 +307,7 @@ class TestSQLConnection(ZuulDBTestCase):
|
|||
self.orderedRelease()
|
||||
self.waitUntilSettled()
|
||||
|
||||
check_results('resultsdb_mysql')
|
||||
check_results('resultsdb_postgresql')
|
||||
check_results('database')
|
||||
|
||||
def test_multiple_sql_connections(self):
|
||||
"Test putting results in different databases"
|
||||
|
@ -352,36 +353,79 @@ class TestSQLConnection(ZuulDBTestCase):
|
|||
tenant.layout.pipelines['check'].failure_actions,
|
||||
connection_name_2
|
||||
)
|
||||
self.assertIsNone(reporter2) # Explicit SQL reporters are ignored
|
||||
|
||||
conn = self.scheds.first.connections.\
|
||||
connections[connection_name_2].engine.connect()
|
||||
buildsets_resultsdb_failures = conn.execute(sa.sql.select(
|
||||
[reporter2.connection.zuul_buildset_table])).fetchall()
|
||||
[reporter1.connection.zuul_buildset_table])).fetchall()
|
||||
# The failure db should only have 1 buildset failed
|
||||
self.assertEqual(1, len(buildsets_resultsdb_failures))
|
||||
self.assertEqual(2, len(buildsets_resultsdb_failures))
|
||||
|
||||
self.assertEqual(
|
||||
'check', buildsets_resultsdb_failures[0]['pipeline'])
|
||||
'check', buildsets_resultsdb_failures[1]['pipeline'])
|
||||
self.assertEqual('org/project',
|
||||
buildsets_resultsdb_failures[0]['project'])
|
||||
buildsets_resultsdb_failures[1]['project'])
|
||||
self.assertEqual(2,
|
||||
buildsets_resultsdb_failures[0]['change'])
|
||||
buildsets_resultsdb_failures[1]['change'])
|
||||
self.assertEqual(
|
||||
'1', buildsets_resultsdb_failures[0]['patchset'])
|
||||
'1', buildsets_resultsdb_failures[1]['patchset'])
|
||||
self.assertEqual(
|
||||
'FAILURE', buildsets_resultsdb_failures[0]['result'])
|
||||
'FAILURE', buildsets_resultsdb_failures[1]['result'])
|
||||
self.assertEqual('Build failed.',
|
||||
buildsets_resultsdb_failures[0]['message'])
|
||||
buildsets_resultsdb_failures[1]['message'])
|
||||
|
||||
check_results('resultsdb_mysql', 'resultsdb_mysql_failures')
|
||||
check_results('resultsdb_postgresql', 'resultsdb_postgresql_failures')
|
||||
check_results('database', 'resultsdb_failures')
|
||||
|
||||
|
||||
class TestSQLConnectionPrefix(TestSQLConnection):
|
||||
config_file = 'zuul-sql-driver-prefix.conf'
|
||||
class TestSQLConnectionPostgres(TestSQLConnectionMysql):
|
||||
config_file = 'zuul-sql-driver-postgres.conf'
|
||||
|
||||
|
||||
class TestSQLConnectionPrefixMysql(TestSQLConnectionMysql):
|
||||
config_file = 'zuul-sql-driver-prefix-mysql.conf'
|
||||
expected_table_prefix = 'prefix_'
|
||||
|
||||
|
||||
class TestSQLConnectionPrefixPostgres(TestSQLConnectionMysql):
|
||||
config_file = 'zuul-sql-driver-prefix-postgres.conf'
|
||||
expected_table_prefix = 'prefix_'
|
||||
|
||||
|
||||
class TestRequiredSQLConnection(BaseTestCase):
|
||||
config = None
|
||||
connections = None
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.addCleanup(self.stop_connection)
|
||||
|
||||
def setup_connection(self, config_file):
|
||||
self.config = configparser.ConfigParser()
|
||||
self.config.read(os.path.join(FIXTURE_DIR, config_file))
|
||||
|
||||
# Setup databases
|
||||
for section_name in self.config.sections():
|
||||
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
|
||||
section_name, re.I)
|
||||
if not con_match:
|
||||
continue
|
||||
|
||||
if self.config.get(section_name, 'driver') == 'sql':
|
||||
if (self.config.get(section_name, 'dburi') ==
|
||||
'$MYSQL_FIXTURE_DBURI$'):
|
||||
f = MySQLSchemaFixture()
|
||||
self.useFixture(f)
|
||||
self.config.set(section_name, 'dburi', f.dburi)
|
||||
elif (self.config.get(section_name, 'dburi') ==
|
||||
'$POSTGRESQL_FIXTURE_DBURI$'):
|
||||
f = PostgresqlSchemaFixture()
|
||||
self.useFixture(f)
|
||||
self.config.set(section_name, 'dburi', f.dburi)
|
||||
self.connections = zuul.lib.connections.ConnectionRegistry()
|
||||
|
||||
def stop_connection(self):
|
||||
self.connections.stop()
|
||||
|
||||
|
||||
class TestConnectionsBadSQL(ZuulDBTestCase):
|
||||
config_file = 'zuul-sql-driver-bad.conf'
|
||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||
|
|
|
@ -5369,28 +5369,34 @@ For CI problems and help debugging, contact ci@example.org"""
|
|||
tenant.layout.pipelines['gate'].merge_failure_message)
|
||||
|
||||
self.assertEqual(
|
||||
len(tenant.layout.pipelines['check'].merge_failure_actions), 1)
|
||||
len(tenant.layout.pipelines['check'].merge_failure_actions), 2)
|
||||
self.assertEqual(
|
||||
len(tenant.layout.pipelines['gate'].merge_failure_actions), 2)
|
||||
len(tenant.layout.pipelines['gate'].merge_failure_actions), 3)
|
||||
|
||||
self.assertTrue(isinstance(
|
||||
tenant.layout.pipelines['check'].merge_failure_actions[0],
|
||||
tenant.layout.pipelines['check'].merge_failure_actions[1],
|
||||
gerritreporter.GerritReporter))
|
||||
|
||||
self.assertTrue(
|
||||
(
|
||||
isinstance(tenant.layout.pipelines['gate'].
|
||||
merge_failure_actions[0],
|
||||
zuul.driver.smtp.smtpreporter.SMTPReporter) and
|
||||
zuul.driver.sql.sqlreporter.SQLReporter) and
|
||||
isinstance(tenant.layout.pipelines['gate'].
|
||||
merge_failure_actions[1],
|
||||
zuul.driver.smtp.smtpreporter.SMTPReporter) and
|
||||
isinstance(tenant.layout.pipelines['gate'].
|
||||
merge_failure_actions[2],
|
||||
gerritreporter.GerritReporter)
|
||||
) or (
|
||||
isinstance(tenant.layout.pipelines['gate'].
|
||||
merge_failure_actions[0],
|
||||
gerritreporter.GerritReporter) and
|
||||
zuul.driver.sql.sqlreporter.SQLReporter) and
|
||||
isinstance(tenant.layout.pipelines['gate'].
|
||||
merge_failure_actions[1],
|
||||
gerritreporter.GerritReporter) and
|
||||
isinstance(tenant.layout.pipelines['gate'].
|
||||
merge_failure_actions[2],
|
||||
zuul.driver.smtp.smtpreporter.SMTPReporter)
|
||||
)
|
||||
)
|
||||
|
|
|
@ -6140,8 +6140,8 @@ class TestProvidesRequiresBuildset(ZuulTestCase):
|
|||
}])
|
||||
|
||||
|
||||
class TestProvidesRequires(ZuulDBTestCase):
|
||||
config_file = "zuul-sql-driver.conf"
|
||||
class TestProvidesRequiresMysql(ZuulDBTestCase):
|
||||
config_file = "zuul-sql-driver-mysql.conf"
|
||||
|
||||
@simple_layout('layouts/provides-requires.yaml')
|
||||
def test_provides_requires_shared_queue_fast(self):
|
||||
|
@ -6682,6 +6682,10 @@ class TestProvidesRequires(ZuulDBTestCase):
|
|||
1, B.messages[0])
|
||||
|
||||
|
||||
class TestProvidesRequiresPostgres(TestProvidesRequiresMysql):
|
||||
config_file = "zuul-sql-driver-postgres.conf"
|
||||
|
||||
|
||||
class TestForceMergeMissingTemplate(ZuulTestCase):
|
||||
tenant_config_file = "config/force-merge-template/main.yaml"
|
||||
|
||||
|
|
|
@ -13,7 +13,6 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.parse
|
||||
import socket
|
||||
|
@ -62,7 +61,8 @@ class BaseTestWeb(ZuulTestCase):
|
|||
self.test_root,
|
||||
info=zuul.model.WebInfo.fromConfig(
|
||||
self.zuul_ini_config),
|
||||
zk_hosts=self.zk_config))
|
||||
zk_hosts=self.zk_config,
|
||||
fake_sql=self.fake_sql))
|
||||
|
||||
self.executor_server.hold_jobs_in_build = True
|
||||
|
||||
|
@ -88,6 +88,7 @@ class BaseTestWeb(ZuulTestCase):
|
|||
self.waitUntilSettled()
|
||||
|
||||
def get_url(self, url, *args, **kwargs):
|
||||
zuul.web.ZuulWebAPI._tenants.cache_clear()
|
||||
return requests.get(
|
||||
urllib.parse.urljoin(self.base_url, url), *args, **kwargs)
|
||||
|
||||
|
@ -266,11 +267,7 @@ class TestWeb(BaseTestWeb):
|
|||
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
req = urllib.request.Request(
|
||||
"http://127.0.0.1:%s/api/tenants" % self.port)
|
||||
f = urllib.request.urlopen(req)
|
||||
data = f.read().decode('utf8')
|
||||
data = json.loads(data)
|
||||
data = self.get_url("api/tenants").json()
|
||||
|
||||
self.assertEqual('tenant-one', data[0]['name'])
|
||||
self.assertEqual(3, data[0]['projects'])
|
||||
|
@ -1021,7 +1018,7 @@ class TestWebSecrets(BaseTestWeb):
|
|||
|
||||
class TestInfo(ZuulDBTestCase, BaseTestWeb):
|
||||
|
||||
config_file = 'zuul-sql-driver.conf'
|
||||
config_file = 'zuul-sql-driver-mysql.conf'
|
||||
|
||||
def setUp(self):
|
||||
super(TestInfo, self).setUp()
|
||||
|
@ -1148,7 +1145,7 @@ class TestGraphiteUrl(TestInfo):
|
|||
|
||||
|
||||
class TestBuildInfo(ZuulDBTestCase, BaseTestWeb):
|
||||
config_file = 'zuul-sql-driver.conf'
|
||||
config_file = 'zuul-sql-driver-mysql.conf'
|
||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||
|
||||
def test_web_list_builds(self):
|
||||
|
@ -1262,7 +1259,7 @@ class TestBuildInfo(ZuulDBTestCase, BaseTestWeb):
|
|||
|
||||
|
||||
class TestArtifacts(ZuulDBTestCase, BaseTestWeb, AnsibleZuulTestCase):
|
||||
config_file = 'zuul-sql-driver.conf'
|
||||
config_file = 'zuul-sql-driver-mysql.conf'
|
||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||
|
||||
def test_artifacts(self):
|
||||
|
@ -2259,7 +2256,7 @@ class TestTenantScopedWebApiTokenWithExpiry(BaseTestWeb):
|
|||
|
||||
|
||||
class TestHeldAttributeInBuildInfo(ZuulDBTestCase, BaseTestWeb):
|
||||
config_file = 'zuul-sql-driver.conf'
|
||||
config_file = 'zuul-sql-driver-mysql.conf'
|
||||
tenant_config_file = 'config/sql-driver/main.yaml'
|
||||
|
||||
def test_autohold_and_retrieve_held_build_info(self):
|
||||
|
|
|
@ -164,9 +164,11 @@ class ZuulApp(object):
|
|||
logging_config.setDebug()
|
||||
logging_config.apply()
|
||||
|
||||
def configure_connections(self, source_only=False, include_drivers=None):
|
||||
def configure_connections(self, source_only=False, include_drivers=None,
|
||||
require_sql=False):
|
||||
self.connections = zuul.lib.connections.ConnectionRegistry()
|
||||
self.connections.configure(self.config, source_only, include_drivers)
|
||||
self.connections.configure(self.config, source_only, include_drivers,
|
||||
require_sql)
|
||||
|
||||
|
||||
class ZuulDaemonApp(ZuulApp, metaclass=abc.ABCMeta):
|
||||
|
|
|
@ -160,7 +160,7 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
|
|||
tls_key=zookeeper_tls_key,
|
||||
tls_ca=zookeeper_tls_ca)
|
||||
|
||||
self.configure_connections()
|
||||
self.configure_connections(require_sql=True)
|
||||
self.sched.setExecutor(gearman)
|
||||
self.sched.setMerger(merger)
|
||||
self.sched.setNodepool(nodepool)
|
||||
|
|
|
@ -124,7 +124,8 @@ class WebServer(zuul.cmd.ZuulDaemonApp):
|
|||
include_drivers=[zuul.driver.sql.SQLDriver,
|
||||
zuul.driver.github.GithubDriver,
|
||||
zuul.driver.pagure.PagureDriver,
|
||||
zuul.driver.gitlab.GitlabDriver])
|
||||
zuul.driver.gitlab.GitlabDriver],
|
||||
require_sql=True)
|
||||
self.configure_authenticators()
|
||||
self._run()
|
||||
except Exception:
|
||||
|
|
|
@ -24,6 +24,7 @@ import subprocess
|
|||
|
||||
import voluptuous as vs
|
||||
|
||||
from zuul.driver.sql.sqlconnection import SQLConnection
|
||||
from zuul import model
|
||||
from zuul.lib import yamlutil as yaml
|
||||
import zuul.manager.dependent
|
||||
|
@ -1272,11 +1273,24 @@ class PipelineParser(object):
|
|||
reporter_set = []
|
||||
allowed_reporters = self.pcontext.tenant.allowed_reporters
|
||||
if conf.get(conf_key):
|
||||
if conf_key in ['success', 'failure', 'merge-failure']:
|
||||
# SQL reporters are required (an implied SQL reporter is
|
||||
# added to every pipeline, ...(1)
|
||||
sql_reporter = self.pcontext.connections\
|
||||
.getSqlReporter(pipeline)
|
||||
sql_reporter.setAction(conf_key)
|
||||
reporter_set.append(sql_reporter)
|
||||
for reporter_name, params \
|
||||
in conf.get(conf_key).items():
|
||||
if allowed_reporters is not None and \
|
||||
reporter_name not in allowed_reporters:
|
||||
raise UnknownConnection(reporter_name)
|
||||
if type(self.pcontext.connections
|
||||
.connections[reporter_name]) == SQLConnection:
|
||||
# (1)... explicit SQL reporters are ignored)
|
||||
self.log.warning("Ignoring SQL reporter configured in"
|
||||
" pipeline %s" % pipeline.name)
|
||||
continue
|
||||
reporter = self.pcontext.connections.getReporter(
|
||||
reporter_name, pipeline, params)
|
||||
reporter.setAction(conf_key)
|
||||
|
@ -1287,6 +1301,16 @@ class PipelineParser(object):
|
|||
if not pipeline.merge_failure_actions:
|
||||
pipeline.merge_failure_actions = pipeline.failure_actions
|
||||
|
||||
for conf_key, action in self.reporter_actions.items():
|
||||
if conf_key in ['success', 'failure', 'merge-failure']\
|
||||
and not getattr(pipeline, action):
|
||||
# SQL reporters are required ... add SQL reporters to the
|
||||
# rest of actions.
|
||||
sql_reporter = self.pcontext.connections\
|
||||
.getSqlReporter(pipeline)
|
||||
sql_reporter.setAction(conf_key)
|
||||
setattr(pipeline, action, [sql_reporter])
|
||||
|
||||
pipeline.disable_at = conf.get(
|
||||
'disable-after-consecutive-failures', None)
|
||||
|
||||
|
|
|
@ -22,28 +22,9 @@ class SQLDriver(Driver, ConnectionInterface, ReporterInterface):
|
|||
name = 'sql'
|
||||
|
||||
def __init__(self):
|
||||
self.tenant_connections = {}
|
||||
cpb.capabilities_registry.register_capabilities(
|
||||
'job_history', True)
|
||||
|
||||
def reconfigure(self, tenant):
|
||||
# NOTE(corvus): This stores the connection of the first
|
||||
# reporter seen for each tenant; we should figure out how to
|
||||
# support multiple connections for a tenant (how do we deal
|
||||
# with pagination of queries across multiple connections), or
|
||||
# otherwise, require there only be one connection in a tenant.
|
||||
if tenant.name in self.tenant_connections:
|
||||
del self.tenant_connections[tenant.name]
|
||||
for pipeline in tenant.layout.pipelines.values():
|
||||
reporters = (pipeline.start_actions + pipeline.success_actions
|
||||
+ pipeline.failure_actions
|
||||
+ pipeline.merge_failure_actions)
|
||||
for reporter in reporters:
|
||||
if not isinstance(reporter, sqlreporter.SQLReporter):
|
||||
continue
|
||||
self.tenant_connections[tenant.name] = reporter.connection
|
||||
return
|
||||
|
||||
def registerScheduler(self, scheduler):
|
||||
self.sched = scheduler
|
||||
|
||||
|
|
|
@ -190,6 +190,8 @@ class SQLConnection(BaseConnection):
|
|||
self.connection = None
|
||||
self.tables_established = False
|
||||
self.table_prefix = self.connection_config.get('table_prefix', '')
|
||||
self.log.info("Initializing SQL connection {} (prefix: {})".format(
|
||||
connection_name, self.table_prefix))
|
||||
|
||||
try:
|
||||
self.dburi = self.connection_config.get('dburi')
|
||||
|
|
|
@ -17,6 +17,9 @@ import re
|
|||
from collections import OrderedDict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from zuul import model
|
||||
from zuul.driver.sql.sqlconnection import SQLConnection
|
||||
from zuul.driver.sql.sqlreporter import SQLReporter
|
||||
import zuul.driver.zuul
|
||||
import zuul.driver.gerrit
|
||||
import zuul.driver.git
|
||||
|
@ -85,10 +88,18 @@ class ConnectionRegistry(object):
|
|||
for driver in self.drivers.values():
|
||||
driver.stop()
|
||||
|
||||
def configure(self, config, source_only=False, include_drivers=None):
|
||||
def configure(self, config, source_only=False, include_drivers=None,
|
||||
require_sql=False):
|
||||
# Register connections from the config
|
||||
connections = OrderedDict()
|
||||
|
||||
if 'database' in config.sections() and not source_only:
|
||||
driver = self.drivers['sql']
|
||||
con_config = dict(config.items('database'))
|
||||
|
||||
connection = driver.getConnection('database', con_config)
|
||||
connections['database'] = connection
|
||||
|
||||
for section_name in config.sections():
|
||||
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
|
||||
section_name, re.I)
|
||||
|
@ -126,6 +137,12 @@ class ConnectionRegistry(object):
|
|||
|
||||
connection = driver.getConnection(con_name, con_config)
|
||||
connections[con_name] = connection
|
||||
if con_driver == 'sql' and 'database' not in connections:
|
||||
# The [database] section was missing. To stay backwards
|
||||
# compatible duplicate the first database connection to the
|
||||
# connection named 'database'
|
||||
connections['database'] = driver.getConnection(
|
||||
'database', con_config)
|
||||
|
||||
# If the [gerrit] or [smtp] sections still exist, load them in as a
|
||||
# connection named 'gerrit' or 'smtp' respectfully
|
||||
|
@ -160,8 +177,38 @@ class ConnectionRegistry(object):
|
|||
connections[driver.name] = DefaultConnection(
|
||||
driver, driver.name, {})
|
||||
|
||||
if require_sql:
|
||||
if 'database' not in connections:
|
||||
raise Exception("Database configuration is required")
|
||||
|
||||
self.connections = connections
|
||||
|
||||
def getSqlConnection(self) -> SQLConnection:
|
||||
"""
|
||||
Gets the SQL connection. This is either the connection
|
||||
described in the [database] section, or the first configured
|
||||
connection.
|
||||
|
||||
:return: The SQL connection.
|
||||
|
||||
"""
|
||||
connection = self.connections.get('database')
|
||||
if not connection:
|
||||
raise Exception("No SQL connections")
|
||||
return connection
|
||||
|
||||
def getSqlReporter(self, pipeline: model.Pipeline) -> SQLReporter:
|
||||
"""
|
||||
Gets the SQL reporter. Such reporter is based on
|
||||
`getSqlConnection`.
|
||||
|
||||
:param pipeline: Pipeline
|
||||
:return: The SQL reporter
|
||||
|
||||
"""
|
||||
connection = self.getSqlConnection()
|
||||
return connection.driver.getReporter(connection, pipeline)
|
||||
|
||||
def getSource(self, connection_name):
|
||||
connection = self.connections[connection_name]
|
||||
return connection.driver.getSource(connection)
|
||||
|
|
|
@ -2423,8 +2423,7 @@ class QueueItem(object):
|
|||
self.log.debug("Checking DB for requirements")
|
||||
requirements_tuple = tuple(sorted(requirements))
|
||||
if requirements_tuple not in self._cached_sql_results:
|
||||
sql_driver = self.pipeline.manager.sched.connections.drivers['sql']
|
||||
conn = sql_driver.tenant_connections.get(self.pipeline.tenant.name)
|
||||
conn = self.pipeline.manager.sched.connections.getSqlConnection()
|
||||
if conn:
|
||||
builds = conn.getBuilds(
|
||||
tenant=self.pipeline.tenant.name,
|
||||
|
|
|
@ -185,7 +185,6 @@ class RPCListener(RPCListenerBase):
|
|||
'get_running_jobs',
|
||||
'get_job_log_stream_address',
|
||||
'tenant_list',
|
||||
'tenant_sql_connection',
|
||||
'status_get',
|
||||
'job_get',
|
||||
'job_list',
|
||||
|
@ -376,16 +375,6 @@ class RPCListener(RPCListenerBase):
|
|||
'queue': queue_size})
|
||||
job.sendWorkComplete(json.dumps(output))
|
||||
|
||||
def handle_tenant_sql_connection(self, job):
|
||||
args = json.loads(job.arguments)
|
||||
sql_driver = self.sched.connections.drivers['sql']
|
||||
conn = sql_driver.tenant_connections.get(args['tenant'])
|
||||
if conn:
|
||||
name = conn.connection_name
|
||||
else:
|
||||
name = ''
|
||||
job.sendWorkComplete(json.dumps(name))
|
||||
|
||||
def handle_status_get(self, job):
|
||||
args = json.loads(job.arguments)
|
||||
output = self.sched.formatStatusJSON(args.get("tenant"))
|
||||
|
|
|
@ -12,10 +12,10 @@
|
|||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import cherrypy
|
||||
import socket
|
||||
|
||||
from cachetools.func import ttl_cache
|
||||
from ws4py.server.cherrypyserver import WebSocketPlugin, WebSocketTool
|
||||
from ws4py.websocket import WebSocket
|
||||
import codecs
|
||||
|
@ -702,11 +702,15 @@ class ZuulWebAPI(object):
|
|||
admin_tenants = json.loads(job.data[0])
|
||||
return admin_tenants
|
||||
|
||||
@ttl_cache(ttl=60)
|
||||
def _tenants(self):
|
||||
job = self.rpc.submitJob('zuul:tenant_list', {})
|
||||
return json.loads(job.data[0])
|
||||
|
||||
@cherrypy.expose
|
||||
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
|
||||
def tenants(self):
|
||||
job = self.rpc.submitJob('zuul:tenant_list', {})
|
||||
ret = json.loads(job.data[0])
|
||||
ret = self._tenants()
|
||||
resp = cherrypy.response
|
||||
resp.headers['Access-Control-Allow-Origin'] = '*'
|
||||
return ret
|
||||
|
@ -963,16 +967,8 @@ class ZuulWebAPI(object):
|
|||
})
|
||||
return ret
|
||||
|
||||
def _get_connection(self, tenant):
|
||||
# Ask the scheduler which sql connection to use for this tenant
|
||||
job = self.rpc.submitJob('zuul:tenant_sql_connection',
|
||||
{'tenant': tenant})
|
||||
connection_name = json.loads(job.data[0])
|
||||
|
||||
if not connection_name:
|
||||
raise cherrypy.HTTPError(404, 'Tenant %s does not exist.' % tenant)
|
||||
|
||||
return self.zuulweb.connections.connections[connection_name]
|
||||
def _get_connection(self):
|
||||
return self.zuulweb.connections.connections['database']
|
||||
|
||||
@cherrypy.expose
|
||||
@cherrypy.tools.save_params()
|
||||
|
@ -981,7 +977,10 @@ class ZuulWebAPI(object):
|
|||
branch=None, patchset=None, ref=None, newrev=None,
|
||||
uuid=None, job_name=None, voting=None, node_name=None,
|
||||
result=None, final=None, held=None, limit=50, skip=0):
|
||||
connection = self._get_connection(tenant)
|
||||
connection = self._get_connection()
|
||||
|
||||
if tenant not in [x['name'] for x in self._tenants()]:
|
||||
raise cherrypy.HTTPError(404, 'Tenant %s does not exist.' % tenant)
|
||||
|
||||
# If final is None, we return all builds, both final and non-final
|
||||
if final is not None:
|
||||
|
@ -1001,7 +1000,7 @@ class ZuulWebAPI(object):
|
|||
@cherrypy.tools.save_params()
|
||||
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
|
||||
def build(self, tenant, uuid):
|
||||
connection = self._get_connection(tenant)
|
||||
connection = self._get_connection()
|
||||
|
||||
data = connection.getBuilds(tenant=tenant, uuid=uuid, limit=1)
|
||||
if not data:
|
||||
|
@ -1040,7 +1039,7 @@ class ZuulWebAPI(object):
|
|||
@cherrypy.expose
|
||||
@cherrypy.tools.save_params()
|
||||
def badge(self, tenant, project=None, pipeline=None, branch=None):
|
||||
connection = self._get_connection(tenant)
|
||||
connection = self._get_connection()
|
||||
|
||||
buildsets = connection.getBuildsets(
|
||||
tenant=tenant, project=project, pipeline=pipeline,
|
||||
|
@ -1063,7 +1062,7 @@ class ZuulWebAPI(object):
|
|||
def buildsets(self, tenant, project=None, pipeline=None, change=None,
|
||||
branch=None, patchset=None, ref=None, newrev=None,
|
||||
uuid=None, result=None, limit=50, skip=0):
|
||||
connection = self._get_connection(tenant)
|
||||
connection = self._get_connection()
|
||||
|
||||
buildsets = connection.getBuildsets(
|
||||
tenant=tenant, project=project, pipeline=pipeline, change=change,
|
||||
|
@ -1079,7 +1078,7 @@ class ZuulWebAPI(object):
|
|||
@cherrypy.tools.save_params()
|
||||
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
|
||||
def buildset(self, tenant, uuid):
|
||||
connection = self._get_connection(tenant)
|
||||
connection = self._get_connection()
|
||||
|
||||
data = connection.getBuildset(tenant, uuid)
|
||||
if not data:
|
||||
|
|
Loading…
Reference in New Issue