Add a config option to disable cron triggers

* Cron triggers (implemented in periodic.py) are always enabled
  although in many installations they are not needed. When they
  are enabled they consume resoruces (keep polling DB etc.).
  This patch add the config option "enabled" under "cron_trigger"
  group that can be used to disable the entire subsystem.
* Wiped out "disable_cron_trigger_thread" pecan app config variable
  that was use to disable cron triggers in unit tests in favor of
  the new config option
* Other minor style changes

Closes-bug: #1724147
Change-Id: I79b9ccb2f4286b3ea8696b7cd65472c8a49937bf
This commit is contained in:
Renat Akhmerov 2017-10-17 14:09:36 +07:00 committed by Mike Fedosin
parent a741d03fad
commit 6efe094012
10 changed files with 132 additions and 198 deletions

View File

@ -53,7 +53,9 @@ def setup_app(config=None):
db_api_v2.setup_db()
if not app_conf.pop('disable_cron_trigger_thread', False):
# TODO(rakhmerov): Why do we run cron triggers in the API layer?
# Should we move it to engine?s
if cfg.CONF.cron_trigger.enabled:
periodic.setup()
coordination.Service('api_group').register_membership()

View File

@ -219,6 +219,18 @@ scheduler_opts = [
),
]
cron_trigger_opts = [
cfg.BoolOpt(
'enabled',
default=True,
help=(
'If this value is set to False then the subsystem of cron triggers'
' is disabled. Disabling cron triggers increases system'
' performance.'
)
),
]
event_engine_opts = [
cfg.HostAddressOpt(
'host',
@ -369,6 +381,7 @@ API_GROUP = 'api'
ENGINE_GROUP = 'engine'
EXECUTOR_GROUP = 'executor'
SCHEDULER_GROUP = 'scheduler'
CRON_TRIGGER_GROUP = 'cron_trigger'
EVENT_ENGINE_GROUP = 'event_engine'
PECAN_GROUP = 'pecan'
COORDINATION_GROUP = 'coordination'
@ -377,6 +390,7 @@ PROFILER_GROUP = profiler.list_opts()[0][0]
KEYCLOAK_OIDC_GROUP = "keycloak_oidc"
OPENSTACK_ACTIONS_GROUP = 'openstack_actions'
CONF.register_opt(wf_trace_log_name_opt)
CONF.register_opt(auth_type_opt)
CONF.register_opt(js_impl_opt)
@ -388,6 +402,7 @@ CONF.register_opts(api_opts, group=API_GROUP)
CONF.register_opts(engine_opts, group=ENGINE_GROUP)
CONF.register_opts(executor_opts, group=EXECUTOR_GROUP)
CONF.register_opts(scheduler_opts, group=SCHEDULER_GROUP)
CONF.register_opts(cron_trigger_opts, group=CRON_TRIGGER_GROUP)
CONF.register_opts(
execution_expiration_policy_opts,
group=EXECUTION_EXPIRATION_POLICY_GROUP
@ -406,8 +421,14 @@ CLI_OPTS = [
default_group_opts = itertools.chain(
CLI_OPTS,
[wf_trace_log_name_opt, auth_type_opt, js_impl_opt, rpc_impl_opt,
rpc_response_timeout_opt, expiration_token_duration]
[
wf_trace_log_name_opt,
auth_type_opt,
js_impl_opt,
rpc_impl_opt,
rpc_response_timeout_opt,
expiration_token_duration
]
)
CONF.register_cli_opts(CLI_OPTS)

View File

@ -37,16 +37,20 @@ _periodic_tasks = {}
class MistralPeriodicTasks(periodic_task.PeriodicTasks):
@periodic_task.periodic_task(spacing=1, run_immediately=True)
def process_cron_triggers_v2(self, ctx):
LOG.debug("Processing cron triggers...")
for trigger in triggers.get_next_cron_triggers():
LOG.debug("Processing cron trigger: %s", trigger)
try:
# Setup admin context before schedule triggers.
ctx = security.create_context(
trigger.trust_id, trigger.project_id
trigger.trust_id,
trigger.project_id
)
auth_ctx.set_ctx(ctx)
LOG.debug("Cron trigger security context: %s", ctx)
# Try to advance the cron trigger next_execution_time and
@ -158,6 +162,7 @@ def setup():
def stop_all_periodic_tasks():
for pt, tg in _periodic_tasks.items():
for tg in _periodic_tasks.values():
tg.stop()
del _periodic_tasks[pt]
_periodic_tasks.clear()

View File

@ -13,41 +13,27 @@
# limitations under the License.
import mock
from oslo_config import cfg
import pecan
import pecan.testing
from webtest import app as webtest_app
from mistral.api import app as pecan_app
from mistral.services import periodic
from mistral.tests.unit import base
from mistral.tests.unit.mstrlfixtures import policy_fixtures
# Disable authentication for API tests.
cfg.CONF.set_default('auth_enable', False, group='pecan')
class APITest(base.DbTestCase):
def setUp(self):
super(APITest, self).setUp()
pecan_opts = cfg.CONF.pecan
self.override_config('auth_enable', False, group='pecan')
self.override_config('enabled', False, group='cron_trigger')
self.app = pecan.testing.load_test_app({
'app': {
'root': pecan_opts.root,
'modules': pecan_opts.modules,
'debug': pecan_opts.debug,
'auth_enable': False,
'disable_cron_trigger_thread': True
}
})
self.addCleanup(pecan.set_config, {}, overwrite=True)
self.addCleanup(cfg.CONF.set_default,
'auth_enable',
False,
group='pecan')
self.app = pecan.testing.load_test_app(
dict(pecan_app.get_pecan_config())
)
# Adding cron trigger thread clean up explicitly in case if
# new tests will provide an alternative configuration for pecan

View File

@ -11,12 +11,12 @@
# limitations under the License.
import datetime
from oslo_config import cfg
from oslo_utils import timeutils
from oslo_utils import uuidutils
import pecan
import pecan.testing
from mistral.api import app as pecan_app
from mistral.tests.unit.api import base
@ -65,14 +65,9 @@ class TestKeystoneMiddleware(base.APITest):
def setUp(self):
super(TestKeystoneMiddleware, self).setUp()
cfg.CONF.set_default('auth_enable', True, group='pecan')
self.override_config('auth_enable', True, group='pecan')
self.override_config('enabled', False, group='cron_trigger')
self.app = pecan.testing.load_test_app({
'app': {
'root': cfg.CONF.pecan.root,
'modules': cfg.CONF.pecan.modules,
'debug': cfg.CONF.pecan.debug,
'auth_enable': cfg.CONF.pecan.auth_enable,
'disable_cron_trigger_thread': True
}
})
self.app = pecan.testing.load_test_app(
dict(pecan_app.get_pecan_config())
)

View File

@ -15,95 +15,66 @@
import mock
from oslo_concurrency import processutils
from oslo_config import cfg
import pecan
from mistral.api import service
from mistral.tests.unit import base
class TestWSGIService(base.BaseTest):
@mock.patch('mistral.api.app.setup_app')
def setUp(self):
super(TestWSGIService, self).setUp()
self.override_config('enabled', False, group='cron_trigger')
@mock.patch.object(service.wsgi, 'Server')
def test_workers_set_default(self, wsgi_server, mock_app):
def test_workers_set_default(self, wsgi_server):
service_name = "mistral_api"
mock_app.return_value = pecan.testing.load_test_app({
'app': {
'root': cfg.CONF.pecan.root,
'modules': cfg.CONF.pecan.modules,
'debug': cfg.CONF.pecan.debug,
'auth_enable': cfg.CONF.pecan.auth_enable,
'disable_cron_trigger_thread': True
}
})
test_service = service.WSGIService(service_name)
wsgi_server.assert_called_once_with(
cfg.CONF,
service_name,
test_service.app,
host='0.0.0.0',
port=8989,
use_ssl=False
)
with mock.patch('mistral.api.app.setup_app'):
test_service = service.WSGIService(service_name)
@mock.patch('mistral.api.app.setup_app')
@mock.patch.object(service.wsgi, 'Server')
def test_workers_set_correct_setting(self, wsgi_server, mock_app):
wsgi_server.assert_called_once_with(
cfg.CONF,
service_name,
test_service.app,
host='0.0.0.0',
port=8989,
use_ssl=False
)
def test_workers_set_correct_setting(self):
self.override_config('api_workers', 8, group='api')
mock_app.return_value = pecan.testing.load_test_app({
'app': {
'root': cfg.CONF.pecan.root,
'modules': cfg.CONF.pecan.modules,
'debug': cfg.CONF.pecan.debug,
'auth_enable': cfg.CONF.pecan.auth_enable,
'disable_cron_trigger_thread': True
}
})
test_service = service.WSGIService("mistral_api")
with mock.patch('mistral.api.app.setup_app'):
test_service = service.WSGIService("mistral_api")
self.assertEqual(8, test_service.workers)
self.assertEqual(8, test_service.workers)
@mock.patch('mistral.api.app.setup_app')
@mock.patch.object(service.wsgi, 'Server')
def test_workers_set_zero_setting(self, wsgi_server, mock_app):
def test_workers_set_zero_setting(self):
self.override_config('api_workers', 0, group='api')
mock_app.return_value = pecan.testing.load_test_app({
'app': {
'root': cfg.CONF.pecan.root,
'modules': cfg.CONF.pecan.modules,
'debug': cfg.CONF.pecan.debug,
'auth_enable': cfg.CONF.pecan.auth_enable,
'disable_cron_trigger_thread': True
}
})
test_service = service.WSGIService("mistral_api")
with mock.patch('mistral.api.app.setup_app'):
test_service = service.WSGIService("mistral_api")
self.assertEqual(processutils.get_worker_count(), test_service.workers)
self.assertEqual(
processutils.get_worker_count(),
test_service.workers
)
@mock.patch('mistral.api.app.setup_app')
@mock.patch.object(service.wsgi, 'Server')
def test_wsgi_service_with_ssl_enabled(self, wsgi_server, mock_app):
def test_wsgi_service_with_ssl_enabled(self, wsgi_server):
self.override_config('enable_ssl_api', True, group='api')
mock_app.return_value = pecan.testing.load_test_app({
'app': {
'root': cfg.CONF.pecan.root,
'modules': cfg.CONF.pecan.modules,
'debug': cfg.CONF.pecan.debug,
'auth_enable': cfg.CONF.pecan.auth_enable,
'disable_cron_trigger_thread': True
}
})
service_name = 'mistral_api'
srv = service.WSGIService(service_name)
wsgi_server.assert_called_once_with(
cfg.CONF,
service_name,
srv.app,
host='0.0.0.0',
port=8989,
use_ssl=True
)
with mock.patch('mistral.api.app.setup_app'):
srv = service.WSGIService(service_name)
wsgi_server.assert_called_once_with(
cfg.CONF,
service_name,
srv.app,
host='0.0.0.0',
port=8989,
use_ssl=True
)

View File

@ -14,13 +14,13 @@
import datetime
import mock
from oslo_config import cfg
import pecan
import pecan.testing
import requests
import requests_mock
import webob
from mistral.api import app as pecan_app
from mistral.auth import keycloak
from mistral import context
from mistral.db.v2 import api as db_api
@ -87,13 +87,17 @@ class TestKeyCloakOIDCAuth(base.BaseTest):
def setUp(self):
super(TestKeyCloakOIDCAuth, self).setUp()
cfg.CONF.set_default('auth_url', AUTH_URL, group='keycloak_oidc')
self.override_config('auth_url', AUTH_URL, group='keycloak_oidc')
self.auth_handler = keycloak.KeycloakAuthHandler()
def _build_request(self, token):
req = webob.Request.blank("/")
req.headers["x-auth-token"] = token
req.get_response = lambda app: None
return req
@requests_mock.Mocker()
@ -104,12 +108,15 @@ class TestKeyCloakOIDCAuth(base.BaseTest):
"roles": ["role1", "role2"]
}
}
# Imitate successful response from KeyCloak with user claims.
req_mock.get(USER_INFO_ENDPOINT, json=USER_CLAIMS)
req = self._build_request(token)
with mock.patch("jwt.decode", return_value=token):
self.auth_handler.authenticate(req)
self.assertEqual("Confirmed", req.headers["X-Identity-Status"])
self.assertEqual("my_realm", req.headers["X-Project-Id"])
self.assertEqual("role1,role2", req.headers["X-Roles"])
@ -117,6 +124,7 @@ class TestKeyCloakOIDCAuth(base.BaseTest):
def test_no_auth_token(self):
req = webob.Request.blank("/")
self.assertRaises(
exc.UnauthorizedException,
self.auth_handler.authenticate,
@ -125,21 +133,23 @@ class TestKeyCloakOIDCAuth(base.BaseTest):
@requests_mock.Mocker()
def test_no_realm_roles(self, req_mock):
token = {
"iss": "http://localhost:8080/auth/realms/my_realm",
}
token = {"iss": "http://localhost:8080/auth/realms/my_realm"}
# Imitate successful response from KeyCloak with user claims.
req_mock.get(USER_INFO_ENDPOINT, json=USER_CLAIMS)
req = self._build_request(token)
with mock.patch("jwt.decode", return_value=token):
self.auth_handler.authenticate(req)
self.assertEqual("Confirmed", req.headers["X-Identity-Status"])
self.assertEqual("my_realm", req.headers["X-Project-Id"])
self.assertEqual("", req.headers["X-Roles"])
def test_wrong_token_format(self):
req = self._build_request(token="WRONG_FORMAT_TOKEN")
self.assertRaises(
exc.UnauthorizedException,
self.auth_handler.authenticate,
@ -151,6 +161,7 @@ class TestKeyCloakOIDCAuth(base.BaseTest):
token = {
"iss": "http://localhost:8080/auth/realms/my_realm",
}
# Imitate failure response from KeyCloak.
req_mock.get(
USER_INFO_ENDPOINT,
@ -159,6 +170,7 @@ class TestKeyCloakOIDCAuth(base.BaseTest):
)
req = self._build_request(token)
with mock.patch("jwt.decode", return_value=token):
try:
self.auth_handler.authenticate(req)
@ -172,12 +184,12 @@ class TestKeyCloakOIDCAuth(base.BaseTest):
@requests_mock.Mocker()
def test_connection_error(self, req_mock):
token = {
"iss": "http://localhost:8080/auth/realms/my_realm",
}
token = {"iss": "http://localhost:8080/auth/realms/my_realm"}
req_mock.get(USER_INFO_ENDPOINT, exc=requests.ConnectionError)
req = self._build_request(token)
with mock.patch("jwt.decode", return_value=token):
self.assertRaises(
exc.MistralException,
@ -190,30 +202,14 @@ class TestKeyCloakOIDCAuthScenarios(base.DbTestCase):
def setUp(self):
super(TestKeyCloakOIDCAuthScenarios, self).setUp()
cfg.CONF.set_default('auth_enable', True, group='pecan')
cfg.CONF.set_default('auth_type', 'keycloak-oidc')
cfg.CONF.set_default('auth_url', AUTH_URL, group='keycloak_oidc')
self.override_config('enabled', False, group='cron_trigger')
self.override_config('auth_enable', True, group='pecan')
self.override_config('auth_type', 'keycloak-oidc')
self.override_config('auth_url', AUTH_URL, group='keycloak_oidc')
pecan_opts = cfg.CONF.pecan
self.app = pecan.testing.load_test_app({
'app': {
'root': pecan_opts.root,
'modules': pecan_opts.modules,
'debug': pecan_opts.debug,
'auth_enable': True,
'disable_cron_trigger_thread': True
}
})
self.addCleanup(pecan.set_config, {}, overwrite=True)
self.addCleanup(
cfg.CONF.set_default,
'auth_enable',
False,
group='pecan'
self.app = pecan.testing.load_test_app(
dict(pecan_app.get_pecan_config())
)
self.addCleanup(cfg.CONF.set_default, 'auth_type', 'keystone')
# Adding cron trigger thread clean up explicitly in case if
# new tests will provide an alternative configuration for pecan
@ -226,6 +222,7 @@ class TestKeyCloakOIDCAuthScenarios(base.DbTestCase):
)
self.mock_ctx = self.patch_ctx.start()
self.mock_ctx.return_value = self.ctx
self.addCleanup(self.patch_ctx.stop)
self.policy = self.useFixture(policy_fixtures.PolicyFixture())
@ -243,9 +240,7 @@ class TestKeyCloakOIDCAuthScenarios(base.DbTestCase):
}
}
headers = {
'X-Auth-Token': str(token)
}
headers = {'X-Auth-Token': str(token)}
with mock.patch("jwt.decode", return_value=token):
resp = self.app.get('/v2/workflows/123', headers=headers)
@ -266,9 +261,7 @@ class TestKeyCloakOIDCAuthScenarios(base.DbTestCase):
}
}
headers = {
'X-Auth-Token': str(token)
}
headers = {'X-Auth-Token': str(token)}
resp = self.app.get(
'/v2/workflows/123',
@ -301,9 +294,7 @@ class TestKeyCloakOIDCAuthScenarios(base.DbTestCase):
}
}
headers = {
'X-Auth-Token': str(token)
}
headers = {'X-Auth-Token': str(token)}
with mock.patch("jwt.decode", return_value=token):
resp = self.app.get(
@ -324,30 +315,14 @@ class TestKeyCloakOIDCAuthApp(base.DbTestCase):
def setUp(self):
super(TestKeyCloakOIDCAuthApp, self).setUp()
cfg.CONF.set_default('auth_enable', True, group='pecan')
cfg.CONF.set_default('auth_type', 'keycloak-oidc')
cfg.CONF.set_default('auth_url', AUTH_URL, group='keycloak_oidc')
self.override_config('enabled', False, group='cron_trigger')
self.override_config('auth_enable', True, group='pecan')
self.override_config('auth_type', 'keycloak-oidc')
self.override_config('auth_url', AUTH_URL, group='keycloak_oidc')
pecan_opts = cfg.CONF.pecan
self.app = pecan.testing.load_test_app({
'app': {
'root': pecan_opts.root,
'modules': pecan_opts.modules,
'debug': pecan_opts.debug,
'auth_enable': True,
'disable_cron_trigger_thread': True
}
})
self.addCleanup(pecan.set_config, {}, overwrite=True)
self.addCleanup(
cfg.CONF.set_default,
'auth_enable',
False,
group='pecan'
self.app = pecan.testing.load_test_app(
dict(pecan_app.get_pecan_config())
)
self.addCleanup(cfg.CONF.set_default, 'auth_type', 'keystone')
# Adding cron trigger thread clean up explicitly in case if
# new tests will provide an alternative configuration for pecan

View File

@ -50,6 +50,8 @@ class TestMembersController(base.APITest):
def setUp(self):
super(TestMembersController, self).setUp()
self.override_config('auth_enable', True, group='pecan')
wf = db_api.create_workflow_definition(WF_DEFINITION)
global MEMBER_URL, WORKFLOW_MEMBER_ACCEPTED
@ -64,7 +66,7 @@ class TestMembersController(base.APITest):
cfg.CONF.set_default('auth_enable', True, group='pecan')
def test_membership_api_without_auth(self):
cfg.CONF.set_default('auth_enable', False, group='pecan')
self.override_config('auth_enable', False, group='pecan')
resp = self.app.get(MEMBER_URL, expect_errors=True)

View File

@ -101,6 +101,7 @@ class FakeHTTPResponse(object):
class BaseTest(base.BaseTestCase):
def setUp(self):
super(BaseTest, self).setUp()
self.addCleanup(spec_parser.clear_caches)
def register_action_class(self, name, cls, attributes=None, desc=None):
@ -209,6 +210,7 @@ class BaseTest(base.BaseTestCase):
def override_config(self, name, override, group=None):
"""Cleanly override CONF variables."""
cfg.CONF.set_override(name, override, group)
self.addCleanup(cfg.CONF.clear_override, name, group)

View File

@ -13,10 +13,6 @@
# limitations under the License.
import eventlet
import mock
import pecan.testing
from oslo_config import cfg
from mistral.api import service as api_service
from mistral.cmd import launch
@ -27,21 +23,12 @@ class ServiceLauncherTest(base.DbTestCase):
def setUp(self):
super(ServiceLauncherTest, self).setUp()
self.override_config('enabled', False, group='cron_trigger')
launch.reset_server_managers()
@mock.patch('mistral.api.app.setup_app')
@mock.patch.object(api_service.wsgi, 'Server')
def test_launch_all(self, wsgi_server, mock_app):
mock_app.return_value = pecan.testing.load_test_app({
'app': {
'root': cfg.CONF.pecan.root,
'modules': cfg.CONF.pecan.modules,
'debug': cfg.CONF.pecan.debug,
'auth_enable': cfg.CONF.pecan.auth_enable,
'disable_cron_trigger_thread': True
}
})
def test_launch_all(self):
eventlet.spawn(launch.launch_any, launch.LAUNCH_OPTIONS.keys())
for i in range(0, 50):
@ -62,19 +49,7 @@ class ServiceLauncherTest(base.DbTestCase):
self.assertEqual(len(svr_proc_mgr.children.keys()), api_workers)
self.assertEqual(len(svr_thrd_mgr.services.services), 3)
@mock.patch('mistral.api.app.setup_app')
@mock.patch.object(api_service.wsgi, 'Server')
def test_launch_process(self, wsgi_server, mock_app):
mock_app.return_value = pecan.testing.load_test_app({
'app': {
'root': cfg.CONF.pecan.root,
'modules': cfg.CONF.pecan.modules,
'debug': cfg.CONF.pecan.debug,
'auth_enable': cfg.CONF.pecan.auth_enable,
'disable_cron_trigger_thread': True
}
})
def test_launch_process(self):
eventlet.spawn(launch.launch_any, ['api'])
for i in range(0, 50):