diff --git a/doc/source/cli/nova-status.rst b/doc/source/cli/nova-status.rst index 2a62dfbd56c3..27ec6423e5b3 100644 --- a/doc/source/cli/nova-status.rst +++ b/doc/source/cli/nova-status.rst @@ -121,6 +121,9 @@ Upgrade **19.0.0 (Stein)** * Checks for the Placement API are modified to require version 1.30. + * Checks are added for the **nova-consoleauth** service to warn and provide + additional instructions to set **[workarounds]enable_consoleauth = True** + while performing a live/rolling upgrade. See Also ======== diff --git a/nova/cmd/status.py b/nova/cmd/status.py index 6025a4920a47..091fa3918707 100644 --- a/nova/cmd/status.py +++ b/nova/cmd/status.py @@ -575,6 +575,87 @@ class UpgradeCommands(object): return UpgradeCheckResult(UpgradeCheckCode.FAILURE, msg) return UpgradeCheckResult(UpgradeCheckCode.SUCCESS) + def _check_console_auths(self): + """Checks for console usage and warns with info for rolling upgrade. + + Iterates all cells checking to see if the nova-consoleauth service is + non-deleted/non-disabled and whether there are any console token auths + in that cell database. If there is a nova-consoleauth service being + used and no console token auths in the cell database, emit a warning + telling the user to set [workarounds]enable_consoleauth = True if they + are performing a rolling upgrade. + """ + # If we're using cells v1, we don't need to check if the workaround + # needs to be used because cells v1 always uses nova-consoleauth. + # If the operator has already enabled the workaround, we don't need + # to check anything. + if CONF.cells.enable or CONF.workarounds.enable_consoleauth: + return UpgradeCheckResult(UpgradeCheckCode.SUCCESS) + + # We need to check cell0 for nova-consoleauth service records because + # it's possible a deployment could have services stored in the cell0 + # database, if they've defaulted their [database]connection in + # nova.conf to cell0. + meta = MetaData(bind=db_session.get_api_engine()) + cell_mappings = Table('cell_mappings', meta, autoload=True) + mappings = cell_mappings.select().execute().fetchall() + + if not mappings: + # There are no cell mappings so we can't determine this, just + # return a warning. The cellsv2 check would have already failed + # on this. + msg = (_('Unable to check consoles without cell mappings.')) + return UpgradeCheckResult(UpgradeCheckCode.WARNING, msg) + + ctxt = nova_context.get_admin_context() + # If we find a non-deleted, non-disabled nova-consoleauth service in + # any cell, we will assume the deployment is using consoles. + using_consoles = False + for mapping in mappings: + with nova_context.target_cell(ctxt, mapping) as cctxt: + # Check for any non-deleted, non-disabled nova-consoleauth + # service. + meta = MetaData(bind=db_session.get_engine(context=cctxt)) + services = Table('services', meta, autoload=True) + consoleauth_service_record = ( + select([services.c.id]).select_from(services).where(and_( + services.c.binary == 'nova-consoleauth', + services.c.deleted == 0, + services.c.disabled == false())).execute().first()) + if consoleauth_service_record: + using_consoles = True + break + + if using_consoles: + # If the deployment is using consoles, we can only be certain the + # upgrade is complete if each compute service is >= Rocky and + # supports storing console token auths in the database backend. + for mapping in mappings: + # Skip cell0 as no compute services should be in it. + if mapping['uuid'] == cell_mapping_obj.CellMapping.CELL0_UUID: + continue + # Get the minimum nova-compute service version in this + # cell. + with nova_context.target_cell(ctxt, mapping) as cctxt: + min_version = self._get_min_service_version( + cctxt, 'nova-compute') + # We could get None for the minimum version in the case of + # new install where there are no computes. If there are + # compute services, they should all have versions. + if min_version is not None and min_version < 35: + msg = _("One or more cells were found which have " + "nova-compute services older than Rocky. " + "Please set the " + "'[workarounds]enable_consoleauth' " + "configuration option to 'True' on your " + "console proxy host if you are performing a " + "rolling upgrade to enable consoles to " + "function during a partial upgrade.") + return UpgradeCheckResult(UpgradeCheckCode.WARNING, + msg) + + return UpgradeCheckResult(UpgradeCheckCode.SUCCESS) + # The format of the check functions is to return an UpgradeCheckResult # object with the appropriate UpgradeCheckCode and details set. If the # check hits warnings or failures then those should be stored in the @@ -595,6 +676,8 @@ class UpgradeCommands(object): (_('API Service Version'), _check_api_service_version), # Added in Rocky (_('Request Spec Migration'), _check_request_spec_migration), + # Added in Stein (but also useful going back to Rocky) + (_('Console Auths'), _check_console_auths), ) def _get_details(self, upgrade_check_result): diff --git a/nova/tests/unit/cmd/test_status.py b/nova/tests/unit/cmd/test_status.py index afd6e38b8bc5..41a61eaf727c 100644 --- a/nova/tests/unit/cmd/test_status.py +++ b/nova/tests/unit/cmd/test_status.py @@ -1040,3 +1040,177 @@ class TestUpgradeCheckRequestSpecMigration(test.NoDBTestCase): "'nova-manage db online_data_migrations' on each cell " "to create the missing request specs." % self.cell_mappings['cell2'].uuid, result.details) + + +class TestUpgradeCheckConsoles(test.NoDBTestCase): + """Tests for the nova-status upgrade check for consoles.""" + + # We'll setup the database ourselves because we need to use cells fixtures + # for multiple cell mappings. + USES_DB_SELF = True + + # This will create three cell mappings: cell0, cell1 (default) and cell2 + NUMBER_OF_CELLS = 2 + + def setUp(self): + super(TestUpgradeCheckConsoles, self).setUp() + self.output = StringIO() + self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.output)) + # We always need the API DB to be setup. + self.useFixture(nova_fixtures.Database(database='api')) + self.cmd = status.UpgradeCommands() + + @staticmethod + def _create_service_in_cell(ctxt, cell, binary, is_deleted=False, + disabled=False, version=None, + create_token_auth=False): + with context.target_cell(ctxt, cell) as cctxt: + service = objects.Service(context=cctxt, binary=binary, + disabled=disabled, host='dontcare') + if version: + service.version = version + service.create() + + if is_deleted: + service.destroy() + + if create_token_auth: + # We have to create an instance in order to create a token + # auth. + inst = objects.Instance(context=cctxt, + uuid=uuidutils.generate_uuid()) + inst.create() + auth = objects.ConsoleAuthToken(context=cctxt, + console_type='novnc', + host='hostname', port=6080, + instance_uuid=inst.uuid) + auth.authorize(CONF.consoleauth.token_ttl) + + return service + + def test_check_cells_v1_enabled(self): + """This is a 'success' case since the console auths check is + ignored when running cells v1. + """ + self.flags(enable=True, group='cells') + result = self.cmd._check_console_auths() + self.assertEqual(status.UpgradeCheckCode.SUCCESS, result.code) + + def test_check_workaround_enabled(self): + """This is a 'success' case since the console auths check is + ignored when the workaround is already enabled. + """ + self.flags(enable_consoleauth=True, group='workarounds') + result = self.cmd._check_console_auths() + self.assertEqual(status.UpgradeCheckCode.SUCCESS, result.code) + + def test_deleted_disabled_consoleauth(self): + """Tests that services other than nova-consoleauth and deleted/disabled + nova-consoleauth services are filtered out. + """ + self._setup_cells() + ctxt = context.get_admin_context() + + # Create a compute service in cell1. + self._create_service_in_cell(ctxt, self.cell_mappings['cell1'], + 'nova-compute') + # Create a deleted consoleauth service in cell1. + self._create_service_in_cell(ctxt, self.cell_mappings['cell1'], + 'nova-consoleauth', is_deleted=True) + # Create a compute service in cell2. + self._create_service_in_cell(ctxt, self.cell_mappings['cell2'], + 'nova-compute') + # Create a disabled consoleauth service in cell2. + self._create_service_in_cell(ctxt, self.cell_mappings['cell2'], + 'nova-consoleauth', disabled=True) + + result = self.cmd._check_console_auths() + self.assertEqual(status.UpgradeCheckCode.SUCCESS, result.code) + + def test_consoleauth_with_upgrade_not_started(self): + """Tests the scenario where the deployment is using consoles but has no + compute services >= Rocky, i.e. a not started upgrade. + """ + self._setup_cells() + ctxt = context.get_admin_context() + + # Create a deleted consoleauth service in cell1. + self._create_service_in_cell(ctxt, self.cell_mappings['cell1'], + 'nova-consoleauth', is_deleted=True) + # Create a live consoleauth service in cell0. (Asserts we check cell0). + self._create_service_in_cell(ctxt, self.cell_mappings['cell0'], + 'nova-consoleauth') + # Create Queens compute services in the cells. + for cell in ['cell1', 'cell2']: + self._create_service_in_cell(ctxt, self.cell_mappings[cell], + 'nova-compute', version=30) + + result = self.cmd._check_console_auths() + self.assertEqual(status.UpgradeCheckCode.WARNING, result.code) + + def test_consoleauth_with_upgrade_complete(self): + """Tests the scenario where the deployment is using consoles and has + all compute services >= Rocky in every cell database, i.e. a completed + upgrade. + """ + self._setup_cells() + ctxt = context.get_admin_context() + + # Create a live consoleauth service in cell1 with token auth. + self._create_service_in_cell(ctxt, self.cell_mappings['cell1'], + 'nova-consoleauth', + create_token_auth=True) + + # Create a live consoleauth service in cell2 with token auth. + self._create_service_in_cell(ctxt, self.cell_mappings['cell2'], + 'nova-consoleauth', + create_token_auth=True) + + # Create Rocky compute services in the cells. + for cell in ['cell1', 'cell2']: + self._create_service_in_cell(ctxt, self.cell_mappings[cell], + 'nova-compute', version=35) + + # Create a Queens compute service in cell0. This not actually valid, + # we do it to assert that we skip cell0 when checking service versions. + self._create_service_in_cell(ctxt, self.cell_mappings['cell0'], + 'nova-compute', version=30) + + result = self.cmd._check_console_auths() + self.assertEqual(status.UpgradeCheckCode.SUCCESS, result.code) + + def test_consoleauth_with_upgrade_partial(self): + """Tests the scenario where the deployment is using consoles and has + compute services >= Rocky in at least one, but not all, cell databases, + i.e. a partial upgrade. + """ + self._setup_cells() + ctxt = context.get_admin_context() + + # Create a live consoleauth service in cell1. + self._create_service_in_cell(ctxt, self.cell_mappings['cell1'], + 'nova-consoleauth') + + # Create a live consoleauth service in cell2 with token auth. + self._create_service_in_cell(ctxt, self.cell_mappings['cell2'], + 'nova-consoleauth', + create_token_auth=True) + + # Create a Queens compute service in cell1. + self._create_service_in_cell(ctxt, self.cell_mappings['cell1'], + 'nova-compute', version=30) + + # Create a Rocky compute service in cell2. + self._create_service_in_cell(ctxt, self.cell_mappings['cell2'], + 'nova-compute', version=35) + + result = self.cmd._check_console_auths() + + self.assertEqual(status.UpgradeCheckCode.WARNING, result.code) + self.assertIn("One or more cells were found which have nova-compute " + "services older than Rocky. " + "Please set the '[workarounds]enable_consoleauth' " + "configuration option to 'True' on your console proxy " + "host if you are performing a rolling upgrade to enable " + "consoles to function during a partial upgrade.", + result.details) diff --git a/releasenotes/notes/nova-status-check-consoleauths-618acb3a67f97418.yaml b/releasenotes/notes/nova-status-check-consoleauths-618acb3a67f97418.yaml new file mode 100644 index 000000000000..ff8a6faf5317 --- /dev/null +++ b/releasenotes/notes/nova-status-check-consoleauths-618acb3a67f97418.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + A new check is added to the ``nova-status upgrade check`` CLI to check for + use of the ``nova-consoleauth`` service to warn and provide additional + instructions to set ``[workarounds]enable_consoleauth = True`` while + performing a live/rolling upgrade.