Robustness updates to support application plugins
Changes include: - Supporting viewing overrides for non-applied uploaded only applications - Re-enable sysinv-helm commands to support plugin development - Expand version pattern match when discovering plugins to account for tarball versions produced from the Cengn build - Ensure that applications which are in an apply related state have their plugins enabled when the conductor is restarted. Change-Id: Ic233ec529065de80105a077d83e56afda2f3ee14 Closes-Bug: #1881711 Closes-Bug: #1881454 Signed-off-by: Robert Church <robert.church@windriver.com>
This commit is contained in:
Robert Church
parent
7149c60059
commit
4617f4d0be
|
|
@ -77,13 +77,12 @@ class HelmChartsController(rest.RestController):
|
|||
except exception.HelmOverrideNotFound:
|
||||
user_overrides = None
|
||||
|
||||
system_apps = pecan.request.rpcapi.get_helm_applications(
|
||||
pecan.request.context)
|
||||
if app_name in system_apps:
|
||||
if pecan.request.rpcapi.app_has_system_plugins(pecan.request.context,
|
||||
app_name):
|
||||
# Get any system overrides for system app.
|
||||
try:
|
||||
system_overrides = pecan.request.rpcapi.get_helm_chart_overrides(
|
||||
pecan.request.context, name, namespace)
|
||||
pecan.request.context, app_name, name, namespace)
|
||||
system_overrides = yaml.safe_dump(system_overrides) \
|
||||
if system_overrides else None
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -14,7 +14,10 @@ import sys
|
|||
from oslo_config import cfg
|
||||
from oslo_log import log
|
||||
|
||||
from sysinv.common import constants
|
||||
from sysinv.common import exception
|
||||
from sysinv.common import service
|
||||
from sysinv.conductor import kube_app
|
||||
from sysinv.db import api
|
||||
from sysinv.helm import helm
|
||||
|
||||
|
|
@ -25,35 +28,56 @@ LOG = log.getLogger(__name__)
|
|||
|
||||
def create_app_overrides_action(path, app_name=None, namespace=None):
|
||||
dbapi = api.get_instance()
|
||||
operator = helm.HelmOperator(dbapi=dbapi)
|
||||
system_apps = operator.get_helm_applications()
|
||||
if app_name not in system_apps:
|
||||
|
||||
try:
|
||||
db_app = dbapi.kube_app_get(app_name)
|
||||
except exception.KubeAppNotFound:
|
||||
LOG.info("Application %s not found" % app_name)
|
||||
return
|
||||
|
||||
helm_operator = helm.HelmOperator(dbapi=dbapi)
|
||||
app_operator = kube_app.AppOperator(dbapi, helm_operator)
|
||||
|
||||
if not app_operator.app_has_system_plugins(db_app):
|
||||
LOG.info("Overrides generation for application %s is "
|
||||
"not supported via this command." % app_name)
|
||||
else:
|
||||
operator.generate_helm_application_overrides(path, app_name, mode=None,
|
||||
cnamespace=namespace)
|
||||
if db_app.status == constants.APP_UPLOAD_SUCCESS:
|
||||
app_operator.activate_app_plugins(db_app)
|
||||
helm_operator.generate_helm_application_overrides(
|
||||
path, app_name, mode=None, cnamespace=namespace)
|
||||
app_operator.deactivate_app_plugins(db_app)
|
||||
else:
|
||||
helm_operator.generate_helm_application_overrides(
|
||||
path, app_name, mode=None, cnamespace=namespace)
|
||||
|
||||
|
||||
def create_armada_app_overrides_action(path, app_name=None, namespace=None):
|
||||
dbapi = api.get_instance()
|
||||
operator = helm.HelmOperator(dbapi=dbapi)
|
||||
system_apps = operator.get_helm_applications()
|
||||
if app_name not in system_apps:
|
||||
|
||||
try:
|
||||
db_app = dbapi.kube_app_get(app_name)
|
||||
except exception.KubeAppNotFound:
|
||||
LOG.info("Application %s not found" % app_name)
|
||||
return
|
||||
|
||||
helm_operator = helm.HelmOperator(dbapi=dbapi)
|
||||
app_operator = kube_app.AppOperator(dbapi, helm_operator)
|
||||
|
||||
if not app_operator.app_has_system_plugins(db_app):
|
||||
LOG.info("Overrides generation for application %s is "
|
||||
"not supported via this command." % app_name)
|
||||
else:
|
||||
operator.generate_helm_application_overrides(path, app_name, mode=None,
|
||||
cnamespace=namespace,
|
||||
armada_format=True,
|
||||
armada_chart_info=None,
|
||||
combined=False)
|
||||
|
||||
|
||||
def create_chart_override_action(path, chart_name=None, namespace=None):
|
||||
dbapi = api.get_instance()
|
||||
operator = helm.HelmOperator(dbapi=dbapi)
|
||||
operator.generate_helm_chart_overrides(path, chart_name, namespace)
|
||||
if db_app.status == constants.APP_UPLOAD_SUCCESS:
|
||||
app_operator.activate_app_plugins(db_app)
|
||||
helm_operator.generate_helm_application_overrides(
|
||||
path, app_name, mode=None, cnamespace=namespace,
|
||||
armada_format=True, armada_chart_info=None, combined=False)
|
||||
app_operator.deactivate_app_plugins(db_app)
|
||||
else:
|
||||
helm_operator.generate_helm_application_overrides(
|
||||
path, app_name, mode=None, cnamespace=namespace,
|
||||
armada_format=True, armada_chart_info=None, combined=False)
|
||||
|
||||
|
||||
def add_action_parsers(subparsers):
|
||||
|
|
@ -69,12 +93,6 @@ def add_action_parsers(subparsers):
|
|||
parser.add_argument('app_name', nargs='?')
|
||||
parser.add_argument('namespace', nargs='?')
|
||||
|
||||
parser = subparsers.add_parser('create-chart-overrides')
|
||||
parser.set_defaults(func=create_chart_override_action)
|
||||
parser.add_argument('path', nargs='?')
|
||||
parser.add_argument('chart_name', nargs='?')
|
||||
parser.add_argument('namespace', nargs='?')
|
||||
|
||||
|
||||
CONF.register_cli_opt(
|
||||
cfg.SubCommandOpt('action',
|
||||
|
|
@ -103,10 +121,3 @@ def main():
|
|||
CONF.action.func(CONF.action.path,
|
||||
CONF.action.app_name,
|
||||
CONF.action.namespace)
|
||||
elif CONF.action.name == 'create-chart-overrides':
|
||||
try:
|
||||
CONF.action.func(CONF.action.path,
|
||||
CONF.action.chart_name,
|
||||
CONF.action.namespace)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
|
|
|||
|
|
@ -160,6 +160,21 @@ class AppOperator(object):
|
|||
# applications
|
||||
self._plugins.audit_plugins()
|
||||
|
||||
def activate_app_plugins(self, rpc_app):
|
||||
app = AppOperator.Application(rpc_app)
|
||||
self._plugins.activate_plugins(app)
|
||||
|
||||
def deactivate_app_plugins(self, rpc_app):
|
||||
app = AppOperator.Application(rpc_app)
|
||||
self._plugins.deactivate_plugins(app)
|
||||
|
||||
def app_has_system_plugins(self, rpc_app):
|
||||
app = AppOperator.Application(rpc_app)
|
||||
# TODO(rchurch): Update once apps are decoupled
|
||||
return (app.system_app or
|
||||
app.name == constants.HELM_APP_CERT_MANAGER or
|
||||
app.name == constants.HELM_APP_OIDC_AUTH)
|
||||
|
||||
def _clear_armada_locks(self):
|
||||
lock_name = "{}.{}.{}".format(ARMADA_LOCK_PLURAL,
|
||||
ARMADA_LOCK_GROUP,
|
||||
|
|
@ -3228,7 +3243,8 @@ class PluginHelper(object):
|
|||
# An enabled plugin will have a python path configuration file name with the
|
||||
# following format: stx_app-platform-integ-apps-1.0-8.pth
|
||||
PTH_PREFIX = 'stx_app-'
|
||||
PTH_PATTERN = re.compile("{}/([\w-]+)/(\d+\.\d+-\d+)".format(common.HELM_OVERRIDES_PATH))
|
||||
PTH_PATTERN = re.compile("{}/([\w-]+)/(\d+\.\d+-\d+.*)/plugins".format(
|
||||
common.HELM_OVERRIDES_PATH))
|
||||
|
||||
def __init__(self, dbapi, helm_op):
|
||||
self._dbapi = dbapi
|
||||
|
|
@ -3267,6 +3283,7 @@ class PluginHelper(object):
|
|||
discoverable_pths = glob.glob(pattern)
|
||||
LOG.debug("PluginHelper: Discoverable app plugins: %s" % discoverable_pths)
|
||||
|
||||
# Examine existing pth files to make sure they are still valid
|
||||
for pth in discoverable_pths:
|
||||
with open(pth, 'r') as f:
|
||||
contents = f.readlines()
|
||||
|
|
@ -3286,7 +3303,8 @@ class PluginHelper(object):
|
|||
else:
|
||||
LOG.warning("PluginHelper: Stale plugin pth file "
|
||||
"found %s: Wrong plugin version "
|
||||
"enabled %s." % (pth, ver))
|
||||
"enabled %s != %s." % (
|
||||
pth, ver, app_obj.app_version))
|
||||
except exception.KubeAppNotFound:
|
||||
LOG.warning("PluginHelper: Stale plugin pth file found"
|
||||
" %s: App is not active." % pth)
|
||||
|
|
@ -3300,6 +3318,17 @@ class PluginHelper(object):
|
|||
LOG.info("PluginHelper: Removing invalid plugin pth: %s" % pth)
|
||||
os.remove(pth)
|
||||
|
||||
# Examine existing applications in an applying state and make sure they
|
||||
# are activated
|
||||
apps = self._dbapi.kube_app_get_all()
|
||||
for app in apps:
|
||||
# If the app is in some form of apply the the plugins should be
|
||||
# enabled
|
||||
if app.status in [constants.APP_APPLY_IN_PROGRESS,
|
||||
constants.APP_APPLY_SUCCESS,
|
||||
constants.APP_APPLY_FAILURE]:
|
||||
self.activate_plugins(AppOperator.Application(app))
|
||||
|
||||
def install_plugins(self, app):
|
||||
""" Install application plugins. """
|
||||
|
||||
|
|
@ -3340,9 +3369,15 @@ class PluginHelper(object):
|
|||
"need to remove." % app.sync_plugins_dir)
|
||||
|
||||
def activate_plugins(self, app):
|
||||
pth_fqpn = self._get_pth_fqpn(app)
|
||||
|
||||
# If this isn't an app with plugins or the plugin path is already
|
||||
# active, skip activation
|
||||
if not app.system_app or os.path.isfile(pth_fqpn):
|
||||
return
|
||||
|
||||
# Add a .pth file to a site-packages directory so the plugin is picked
|
||||
# automatically on a conductor restart
|
||||
pth_fqpn = self._get_pth_fqpn(app)
|
||||
with open(pth_fqpn, 'w') as f:
|
||||
f.write(app.sync_plugins_dir + '\n')
|
||||
LOG.info("PluginHelper: Enabled plugin directory %s: created %s" % (
|
||||
|
|
@ -3362,6 +3397,10 @@ class PluginHelper(object):
|
|||
self._helm_op.discover_plugins()
|
||||
|
||||
def deactivate_plugins(self, app):
|
||||
# If the application doesn't have any plugins, skip deactivation
|
||||
if not app.system_app:
|
||||
return
|
||||
|
||||
pth_fqpn = self._get_pth_fqpn(app)
|
||||
if os.path.exists(pth_fqpn):
|
||||
# Remove the pth file, so on a conductor restart this installed
|
||||
|
|
|
|||
|
|
@ -10757,7 +10757,8 @@ class ConductorManager(service.PeriodicService):
|
|||
"""
|
||||
return self._helm.get_helm_chart_namespaces(chart_name)
|
||||
|
||||
def get_helm_chart_overrides(self, context, chart_name, cnamespace=None):
|
||||
def get_helm_chart_overrides(self, context, app_name, chart_name,
|
||||
cnamespace=None):
|
||||
"""Get the overrides for a supported chart.
|
||||
|
||||
This method retrieves overrides for a supported chart. Overrides for
|
||||
|
|
@ -10765,6 +10766,7 @@ class ConductorManager(service.PeriodicService):
|
|||
is requested.
|
||||
|
||||
:param context: request context.
|
||||
:param app_name: name of a supported application
|
||||
:param chart_name: name of a supported chart
|
||||
:param cnamespace: (optional) namespace
|
||||
:returns: dict of overrides.
|
||||
|
|
@ -10794,16 +10796,30 @@ class ConductorManager(service.PeriodicService):
|
|||
}
|
||||
}
|
||||
"""
|
||||
return self._helm.get_helm_chart_overrides(chart_name,
|
||||
cnamespace)
|
||||
|
||||
def get_helm_applications(self, context):
|
||||
"""Get supported applications.
|
||||
app = kubeapp_obj.get_by_name(context, app_name)
|
||||
if app.status in [constants.APP_APPLY_IN_PROGRESS,
|
||||
constants.APP_APPLY_SUCCESS,
|
||||
constants.APP_APPLY_FAILURE]:
|
||||
overrides = self._helm.get_helm_chart_overrides(chart_name,
|
||||
cnamespace)
|
||||
else:
|
||||
self._app.activate_app_plugins(app)
|
||||
overrides = self._helm.get_helm_chart_overrides(chart_name,
|
||||
cnamespace)
|
||||
self._app.deactivate_app_plugins(app)
|
||||
|
||||
:returns: a list of suppotred applications that associated overrides may
|
||||
be provided.
|
||||
return overrides
|
||||
|
||||
def app_has_system_plugins(self, context, app_name):
|
||||
|
||||
"""Determine if the application has system plugin support.
|
||||
|
||||
:returns: True if the application has system plugins and can generate
|
||||
system overrides.
|
||||
"""
|
||||
return self._helm.get_helm_applications()
|
||||
app = kubeapp_obj.get_by_name(context, app_name)
|
||||
return self._app.app_has_system_plugins(app)
|
||||
|
||||
def get_helm_application_namespaces(self, context, app_name):
|
||||
"""Get supported application namespaces.
|
||||
|
|
|
|||
|
|
@ -1608,10 +1608,12 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy):
|
|||
self.make_msg('get_helm_chart_namespaces',
|
||||
chart_name=chart_name))
|
||||
|
||||
def get_helm_chart_overrides(self, context, chart_name, cnamespace=None):
|
||||
def get_helm_chart_overrides(self, context, app_name, chart_name,
|
||||
cnamespace=None):
|
||||
"""Get the overrides for a supported chart.
|
||||
|
||||
:param context: request context.
|
||||
:param app_name: name of a supported application
|
||||
:param chart_name: name of a supported chart
|
||||
:param cnamespace: (optional) namespace
|
||||
:returns: dict of overrides.
|
||||
|
|
@ -1619,18 +1621,20 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy):
|
|||
"""
|
||||
return self.call(context,
|
||||
self.make_msg('get_helm_chart_overrides',
|
||||
app_name=app_name,
|
||||
chart_name=chart_name,
|
||||
cnamespace=cnamespace))
|
||||
|
||||
def get_helm_applications(self, context):
|
||||
def app_has_system_plugins(self, context, app_name):
|
||||
|
||||
"""Get supported applications.
|
||||
"""Determine if the application has system plugin support.
|
||||
|
||||
:returns: a list of suppotred applications that associated overrides may
|
||||
be provided.
|
||||
:returns: True if the application has system plugins and can generate
|
||||
system overrides.
|
||||
"""
|
||||
return self.call(context,
|
||||
self.make_msg('get_helm_applications'))
|
||||
self.make_msg('app_has_system_plugins',
|
||||
app_name=app_name))
|
||||
|
||||
def get_helm_application_namespaces(self, context, app_name):
|
||||
"""Get supported application namespaces.
|
||||
|
|
|
|||
|
|
@ -222,8 +222,8 @@ class HelmOperator(object):
|
|||
|
||||
return supported_helm_applications
|
||||
|
||||
def get_helm_applications(self):
|
||||
""" Get the system applications and charts """
|
||||
def get_active_helm_applications(self):
|
||||
""" Get the active system applications and charts """
|
||||
return self.helm_system_applications
|
||||
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -18,8 +18,9 @@ from sysinv.tests.db import utils as dbutils
|
|||
class FakeConductorAPI(object):
|
||||
|
||||
def __init__(self):
|
||||
self.app_has_system_plugins = mock.MagicMock()
|
||||
self.get_helm_application_namespaces = mock.MagicMock()
|
||||
self.get_helm_applications = mock.MagicMock()
|
||||
self.get_active_helm_applications = mock.MagicMock()
|
||||
self.get_helm_chart_overrides = mock.MagicMock()
|
||||
self.merge_overrides = mock.MagicMock()
|
||||
|
||||
|
|
@ -72,10 +73,11 @@ class ApiHelmChartTestCaseMixin(base.FunctionalTest,
|
|||
chart_namespace='kube-system',
|
||||
system_override_attr={"enabled": False},
|
||||
user_override="global:\n replicas: \"3\"\n")
|
||||
self.fake_helm_apps = self.fake_conductor_api.get_helm_applications
|
||||
self.fake_helm_apps = self.fake_conductor_api.get_active_helm_applications
|
||||
self.fake_ns = self.fake_conductor_api.get_helm_application_namespaces
|
||||
self.fake_override = self.fake_conductor_api.get_helm_chart_overrides
|
||||
self.fake_merge_overrides = self.fake_conductor_api.merge_overrides
|
||||
self.fake_system_app = self.fake_conductor_api.app_has_system_plugins
|
||||
|
||||
def exception_helm_override(self):
|
||||
print('Raised a fake exception')
|
||||
|
|
@ -169,6 +171,7 @@ class ApiHelmChartShowTestSuiteMixin(ApiHelmChartTestCaseMixin):
|
|||
super(ApiHelmChartShowTestSuiteMixin, self).setUp()
|
||||
|
||||
def test_no_system_override(self):
|
||||
self.fake_system_app.return_value = False
|
||||
url = self.get_single_url_helm_override('platform-integ-apps',
|
||||
'ceph-pools-audit', 'kube-system')
|
||||
response = self.get_json(url)
|
||||
|
|
@ -190,6 +193,8 @@ class ApiHelmChartShowTestSuiteMixin(ApiHelmChartTestCaseMixin):
|
|||
response.json['error_message'])
|
||||
|
||||
def test_fetch_helm_override_show_invalid_helm_chart(self):
|
||||
self.fake_system_app.return_value = False
|
||||
|
||||
url = self.get_single_url_helm_override('platform-integ-apps',
|
||||
'invalid_value', 'kube-system')
|
||||
response = self.get_json(url, expect_errors=True)
|
||||
|
|
@ -202,6 +207,7 @@ class ApiHelmChartShowTestSuiteMixin(ApiHelmChartTestCaseMixin):
|
|||
response.json['error_message'])
|
||||
|
||||
def test_fetch_helm_override_show_invalid_namespace(self):
|
||||
self.fake_system_app.return_value = False
|
||||
url = self.get_single_url_helm_override('platform-integ-apps',
|
||||
'ceph-pools-audit',
|
||||
'invalid_value')
|
||||
|
|
@ -287,6 +293,7 @@ class ApiHelmChartDeleteTestSuiteMixin(ApiHelmChartTestCaseMixin):
|
|||
|
||||
# Test that a valid DELETE operation is successful
|
||||
def test_delete_helm_override_success(self):
|
||||
self.fake_system_app.return_value = False
|
||||
|
||||
# Verify that user override exists initially
|
||||
url = self.get_single_url_helm_override('platform-integ-apps',
|
||||
|
|
@ -494,6 +501,7 @@ class ApiHelmChartPatchTestSuiteMixin(ApiHelmChartTestCaseMixin):
|
|||
'set': ['global.replicas=2']}},
|
||||
headers=self.API_HEADERS,
|
||||
expect_errors=True)
|
||||
self.fake_system_app.return_value = False
|
||||
response = self.get_json(url, expect_errors=True)
|
||||
self.assertEqual(response.status_code, http_client.OK)
|
||||
# Verify the values of the response with the values in database
|
||||
|
|
|
|||
Loading…
Reference in New Issue