starlingx
/
nfv
Code Issues Proposed changes

sw-deploy strategy for in-service patch-release(AIO-SX) 

sw-deploy API implementation and create
strategy validations.
The API implementations are common for
both major and minor release.The current
changes are tested only on AIO-SX for patch
release(in-service patch)

New CLI command usage:
sw-manager sw-deploy-strategy create starlingx-24.03.1
sw-manager sw-deploy-strategy apply
sw-manager sw-deploy-strategy abort
sw-manager sw-deploy-strategy delete

Test Plan:
PASSED: On SX system, perform end to end sw-deploy strategy
update using in-service patch.
PASSED: On SX system, provide invalid release as input to
sw-manager sw-deploy-strategy create command.
Create startegy fails.

Story: 2011045
Task: 49911

Change-Id: I265c8a1f9fbadf04275e0af788614d094c23315c
Signed-off-by: Vanathi.Selvaraju <vanathi.selvaraju@windriver.com>
This commit is contained in:
Vanathi.Selvaraju 2024-04-22 11:43:42 -04:00 committed by Vanathi Selvaraju
parent 93fd832c4b
commit 1f05af23e3
10 changed files with 325 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
nfv/nfv-common/nfv_common/tasks/_task_future.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2016 Wind River Systems, Inc.
# Copyright (c) 2015-2016,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -9,6 +9,10 @@ from nfv_common.tasks._task_work import TaskWork
DLOG = debug.debug_get_logger('nfv_common.tasks.task_future')
# TODO(vselvara)The timeout to be changed back to 20sec when the start
# software-api is async for patch-release
TASK_TIMEOUT = 600
class TaskFuture(object):
"""
@ -57,7 +61,7 @@ class TaskFuture(object):
if timeout_in_secs is None:
# WARNING: Any change to the default timeout must be reflected in
# the timeouts used for any work being done.
timeout_in_secs = 20
timeout_in_secs = TASK_TIMEOUT
elif 0 >= timeout_in_secs:
timeout_in_secs = None # No timeout wanted, wait forever

4
nfv/nfv-common/nfv_common/thread/_thread.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2016 Wind River Systems, Inc.
# Copyright (c) 2015-2016,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -94,7 +94,7 @@ class Thread(object):
self._stall_timestamp_ms = \
timers.get_monotonic_timestamp_in_ms()
DLOG.error("Thread %s stalled, progress_marker=%s, "
DLOG.warn("Thread %s stalled, progress_marker=%s, "
"elapsed_secs=%s." % (self._name,
self._progress_marker.value,
self.stall_elapsed_secs))

7
nfv/nfv-plugins/nfv_plugins/nfvi_plugins/config.ini
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2018 Wind River Systems, Inc.
# Copyright (c) 2015-2018,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -117,11 +117,14 @@ port=30004
# WARNING: Any changes to these timeouts must be reflected in the timeouts
# used for the associated REST API calls.
# TODO(vselvara) sysinv and usm timeout to be changed back to 60 once
# the sw-deploy start API for patch release is made asynchronous.
[nfvi-timeouts]
openstack.get_token=10
neutron.disable_host_services=40
neutron.delete_host_services=40
glance.upload_image_data_by_file=180
glance.upload_image_data_by_url=180
sysinv=60
sysinv=600
patching.apply_patch=180
usm=600

100
nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py
View File

@ -2351,8 +2351,16 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
DLOG.error("USM software deploy get release did not complete.")
return
release_info = None
release_data = future.result.data
release_info = release_data["metadata"].get(release, None)
for rel in release_data:
if rel['release_id'] == release:
release_info = rel
break
if not release_info:
DLOG.error("Software release not found.")
return
future.work(usm.sw_deploy_host_list, self._platform_token)
future.result = (yield)
@ -2362,11 +2370,13 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
return
hosts_info_data = future.result.data
if hosts_info_data == []:
hosts_info_data = None
upgrade_obj = nfvi.objects.v1.Upgrade(
release,
release_info,
hosts_info_data["data"],
hosts_info_data,
)
response['result-data'] = upgrade_obj
@ -2415,17 +2425,13 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
future.work(usm.sw_deploy_precheck, self._platform_token, release)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("USM software deploy precheck did not complete.")
return
upgrade_obj = nfvi.objects.v1.Upgrade(
release,
None,
None)
precheck_data = future.result.data['error']
response['result-data'] = upgrade_obj
response['result-data'] = precheck_data
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
@ -2446,7 +2452,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
callback.send(response)
callback.close()
def upgrade_start(self, future, release, callback):
def sw_deploy_start(self, future, release, callback):
"""
Start a USM software deploy
"""
@ -2455,9 +2461,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
response['reason'] = ''
try:
upgrade_data = future.result.data
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
@ -2472,15 +2476,37 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
future.work(usm.sw_deploy_start, self._platform_token, release)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("USM software deploy start did not complete.")
return
# TODO(vselvara): remove the state check here once the api is changed
# to async. state check to be done in _strategy_steps.py
future.work(usm.sw_deploy_host_list, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("USM software deploy host list did not complete.")
return
hosts_info_data = future.result.data
future.work(usm.sw_deploy_show, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("USM software deploy show did not complete.")
return
state_info = future.result.data
if state_info:
state_info = state_info[0]
else:
state_info = None
upgrade_obj = nfvi.objects.v1.Upgrade(
release,
upgrade_data,
None)
state_info,
hosts_info_data,
)
response['result-data'] = upgrade_obj
response['completed'] = True
@ -2503,7 +2529,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
callback.send(response)
callback.close()
def upgrade_activate(self, future, release, callback):
def sw_deploy_activate(self, future, release, callback):
"""
Activate a USM software deployement
"""
@ -2526,17 +2552,29 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
self._platform_token = future.result.data
future.work(usm.sw_deploy_activate, self._platform_token, release)
future.work(usm.sw_deploy_activate, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("USM software deploy activate did not complete.")
return
upgrade_data = future.result.data
future.work(usm.sw_deploy_show, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("USM software deploy show did not complete.")
return
state_info = future.result.data
if state_info:
state_info = state_info[0]
else:
state_info = None
upgrade_obj = nfvi.objects.v1.Upgrade(
release,
upgrade_data,
state_info,
None)
response['result-data'] = upgrade_obj
@ -2560,7 +2598,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
callback.send(response)
callback.close()
def upgrade_complete(self, future, release, callback):
def sw_deploy_complete(self, future, release, callback):
"""
Complete a USM software deployement
"""
@ -2583,17 +2621,34 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
self._platform_token = future.result.data
future.work(usm.sw_deploy_complete, self._platform_token, release)
future.work(usm.sw_deploy_complete, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("USM software deploy complete did not complete.")
return
upgrade_data = future.result.data
future.work(usm.sw_deploy_get_release, self._platform_token, release)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("USM software deploy get release did not complete.")
return
release_info = None
release_data = future.result.data
for rel in release_data:
if rel['release_id'] == release:
release_info = rel
break
if not release_info:
DLOG.error("Software release not found.")
return
upgrade_obj = nfvi.objects.v1.Upgrade(
release,
upgrade_data,
release_info,
None)
response['result-data'] = upgrade_obj
@ -3777,6 +3832,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
future.result = (yield)
if not future.result.is_complete():
DLOG.error("USM software deploy host %s did not complete." % host_name)
return
response['completed'] = True

38
nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/usm.py
View File

@ -13,6 +13,7 @@ from nfv_plugins.nfvi_plugins.openstack.rest_api import rest_api_request
REST_API_REQUEST_TIMEOUT = 60
REST_API_START_REQUEST_TIMEOUT = 600
DLOG = debug.debug_get_logger('nfv_plugins.nfvi_plugins.openstack.usm')
@ -22,7 +23,7 @@ def _usm_api_cmd(token, endpoint):
if base_url is None:
raise ValueError("PlatformService USM URL is invalid")
url = os.path.join(base_url, "v1/software", endpoint)
url = os.path.join(base_url, "v1/", endpoint)
return url
@ -44,7 +45,7 @@ def _api_get(token, url):
return response
def _api_post(token, url, payload, headers=None):
def _api_post(token, url, payload, headers=None, timeout_in_secs=REST_API_REQUEST_TIMEOUT):
"""
Generic POST to an endpoint with a payload
"""
@ -56,7 +57,7 @@ def _api_post(token, url, payload, headers=None):
url,
headers,
json.dumps(payload),
timeout_in_secs=REST_API_REQUEST_TIMEOUT)
timeout_in_secs)
return response
@ -65,7 +66,18 @@ def sw_deploy_get_release(token, release):
Query USM for information about a specific upgrade
"""
uri = f"show/{release}"
uri = f"release" # noqa:F541 pylint: disable=W1309
url = _usm_api_cmd(token, uri)
response = _api_get(token, url)
return response
def sw_deploy_show(token):
"""
Query USM for information about a specific upgrade
"""
uri = f"deploy" # noqa:F541 pylint: disable=W1309
url = _usm_api_cmd(token, uri)
response = _api_get(token, url)
return response
@ -76,8 +88,7 @@ def sw_deploy_host_list(token):
Query USM for information about a hosts during a deployment
"""
# TODO(jkraitbe): This API will change in the future
uri = "host_list"
uri = "deploy_host"
url = _usm_api_cmd(token, uri)
response = _api_get(token, url)
return response
@ -88,7 +99,8 @@ def sw_deploy_precheck(token, release, force=False):
Ask USM to precheck before a deployment
"""
uri = f"deploy_precheck/{release}/force" if force else f"deploy_precheck/{release}"
uri = (f"deploy/{release}/precheck/force" if
force else f"deploy/{release}/precheck")
url = _usm_api_cmd(token, uri)
response = _api_post(token, url, {})
return response
@ -99,9 +111,9 @@ def sw_deploy_start(token, release, force=False):
Ask USM to start a deployment
"""
uri = f"deploy_start/{release}/force" if force else f"deploy_start/{release}"
uri = f"deploy/{release}/start/force" if force else f"deploy/{release}/start"
url = _usm_api_cmd(token, uri)
response = _api_post(token, url, {})
response = _api_post(token, url, {}, None, REST_API_START_REQUEST_TIMEOUT)
return response
@ -116,23 +128,23 @@ def sw_deploy_execute(token, host_name):
return response
def sw_deploy_activate(token, release):
def sw_deploy_activate(token):
"""
Ask USM activate a deployment
"""
uri = f"deploy_activate/{release}"
uri = f"deploy/activate" # noqa:F541 pylint: disable=W1309
url = _usm_api_cmd(token, uri)
response = _api_post(token, url, {})
return response
def sw_deploy_complete(token, release):
def sw_deploy_complete(token):
"""
Ask USM complete a deployment
"""
uri = f"deploy_complete/{release}"
uri = f"deploy/complete" # noqa:F541 pylint: disable=W1309
url = _usm_api_cmd(token, uri)
response = _api_post(token, url, {})
return response

64
nfv/nfv-tests/nfv_unit_tests/tests/test_sw_deploy_strategy.py
View File

@ -172,6 +172,40 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
return controller_hosts, storage_hosts, worker_hosts, strategy
@mock.patch('nfv_common.strategy._strategy.Strategy._build')
def test_sw_deploy_strategy_build_steps(self, fake_build):
"""
Verify build phase steps and stages for sw deploy strategy creation.
"""
# setup a minimal host environment
self.create_host('controller-0', aio=True)
update_obj = SwUpgrade()
strategy = self.create_sw_deploy_strategy(
single_controller=True)
update_obj = SwUpgrade()
strategy.sw_update_obj = update_obj
strategy.build()
# verify the build phase and steps
build_phase = strategy.build_phase.as_dict()
query_steps = [
{'name': 'query-alarms'},
{'name': 'sw-deploy-precheck'},
{'name': 'query-upgrade'},
]
expected_results = {
'total_stages': 1,
'stages': [
{'name': 'sw-upgrade-query',
'total_steps': len(query_steps),
'steps': query_steps,
},
],
}
sw_update_testcase.validate_phase(build_phase, expected_results)
# ~~~ SW-DEPLOY Start ~~~
@mock.patch('nfv_vim.strategy._strategy.get_local_host_name',
@ -205,11 +239,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'total_stages': 1,
'stages': [
{'name': 'sw-upgrade-start',
'total_steps': 4,
'total_steps': 3,
'steps': [
{'name': 'query-alarms'},
{'name': 'sw-deploy-precheck',
'release': release},
{'name': 'start-upgrade',
'release': release},
{'name': 'system-stabilize',
@ -253,11 +285,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'total_stages': 1,
'stages': [
{'name': 'sw-upgrade-start',
'total_steps': 4,
'total_steps': 3,
'steps': [
{'name': 'query-alarms'},
{'name': 'sw-deploy-precheck',
'release': release},
{'name': 'start-upgrade',
'release': release},
{'name': 'system-stabilize',
@ -302,13 +332,11 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'total_stages': 1,
'stages': [
{'name': 'sw-upgrade-start',
'total_steps': 6,
'total_steps': 5,
'steps': [
{'name': 'query-alarms'},
{'name': 'swact-hosts',
'entity_names': ['controller-1']},
{'name': 'sw-deploy-precheck',
'release': release},
{'name': 'start-upgrade',
'release': release},
{'name': 'system-stabilize',
@ -994,10 +1022,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'stages': [
{
'name': 'sw-upgrade-start',
'total_steps': 4,
'total_steps': 3,
'steps': [
{'name': 'query-alarms'},
{'name': 'sw-deploy-precheck', 'release': release},
{'name': 'start-upgrade', 'release': release},
{'name': 'system-stabilize', 'timeout': 60},
],
@ -1060,10 +1087,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'stages': [
{
'name': 'sw-upgrade-start',
'total_steps': 4,
'total_steps': 3,
'steps': [
{'name': 'query-alarms'},
{'name': 'sw-deploy-precheck', 'release': release},
{'name': 'start-upgrade', 'release': release},
{'name': 'system-stabilize', 'timeout': 60},
],
@ -1130,10 +1156,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'stages': [
{
'name': 'sw-upgrade-start',
'total_steps': 4,
'total_steps': 3,
'steps': [
{'name': 'query-alarms'},
{'name': 'sw-deploy-precheck', 'release': release},
{'name': 'start-upgrade', 'release': release},
{'name': 'system-stabilize', 'timeout': 60},
],
@ -1205,10 +1230,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'stages': [
{
'name': 'sw-upgrade-start',
'total_steps': 4,
'total_steps': 3,
'steps': [
{'name': 'query-alarms'},
{'name': 'sw-deploy-precheck', 'release': release},
{'name': 'start-upgrade', 'release': release},
{'name': 'system-stabilize', 'timeout': 60},
],
@ -1292,10 +1316,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'stages': [
{
'name': 'sw-upgrade-start',
'total_steps': 4,
'total_steps': 3,
'steps': [
{'name': 'query-alarms'},
{'name': 'sw-deploy-precheck', 'release': release},
{'name': 'start-upgrade', 'release': release},
{'name': 'system-stabilize', 'timeout': 60},
],
@ -1389,10 +1412,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'stages': [
{
'name': 'sw-upgrade-start',
'total_steps': 4,
'total_steps': 3,
'steps': [
{'name': 'query-alarms'},
{'name': 'sw-deploy-precheck', 'release': release},
{'name': 'start-upgrade', 'release': release},
{'name': 'system-stabilize', 'timeout': 60},
],

8
nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py
View File

@ -1400,11 +1400,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'total_stages': 8,
'stages': [
{'name': 'sw-upgrade-start',
'total_steps': 4,
'total_steps': 3,
'steps': [
{'name': 'query-alarms'},
{'name': 'sw-deploy-precheck',
'release': strategy.nfvi_upgrade.release},
{'name': 'start-upgrade',
'release': strategy.nfvi_upgrade.release},
{'name': 'system-stabilize',
@ -1570,13 +1568,11 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'total_stages': 6,
'stages': [
{'name': 'sw-upgrade-start',
'total_steps': 6,
'total_steps': 5,
'steps': [
{'name': 'query-alarms'},
{'name': 'swact-hosts',
'entity_names': ['controller-1']},
{'name': 'sw-deploy-precheck',
'release': strategy.nfvi_upgrade.release},
{'name': 'start-upgrade',
'release': strategy.nfvi_upgrade.release},
{'name': 'system-stabilize',

4
nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_upgrade.py
View File

@ -33,14 +33,14 @@ class Upgrade(ObjectData):
@property
def state(self):
if self.release_info is None:
if not self.release_info:
return None
return self.release_info["state"]
@property
def reboot_required(self):
if self.release_info is None:
if not self.release_info:
return None
return self.release_info["reboot_required"] == USM_REBOOT_REQUIRED

5
nfv/nfv-vim/nfv_vim/strategy/_strategy.py
View File

@ -1812,6 +1812,7 @@ class SwUpgradeStrategy(
stage = strategy.StrategyStage(strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_QUERY)
stage.add_step(strategy.QueryAlarmsStep(ignore_alarms=self._ignore_alarms))
stage.add_step(strategy.SwDeployPrecheckStep(release=self._release))
stage.add_step(strategy.QueryUpgradeStep(release=self._release))
self.build_phase.add_stage(stage)
@ -1852,7 +1853,6 @@ class SwUpgradeStrategy(
if self.nfvi_upgrade.is_available:
# sw-deploy start must be done on controller-0
self._swact_fix(stage, HOST_NAME.CONTROLLER_1)
stage.add_step(strategy.SwDeployPrecheckStep(release=self._release))
stage.add_step(strategy.UpgradeStartStep(release=self._release))
stage.add_step(strategy.SystemStabilizeStep())
# sw-deploy host must first be on controller-1
@ -1890,8 +1890,7 @@ class SwUpgradeStrategy(
if result in [strategy.STRATEGY_RESULT.SUCCESS,
strategy.STRATEGY_RESULT.DEGRADED]:
if self.nfvi_upgrade.release_info is None:
if not self.nfvi_upgrade.release_info:
reason = "Software release does not exist."
DLOG.warn(reason)
self._state = strategy.STRATEGY_STATE.BUILD_FAILED

302
nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py
View File

@ -24,6 +24,11 @@ KUBE_CERT_UPDATE_TRUSTBOTHCAS = "trust-both-cas"
KUBE_CERT_UPDATE_TRUSTNEWCA = "trust-new-ca"
KUBE_CERT_UPDATE_UPDATECERTS = "update-certs"
# sw-deploy strategy constants
SW_DEPLOY_START = 'start-done'
SW_HOST_DEPLOYED = 'deployed'
SW_DEPLOY_ACTIVATE_DONE = 'activate-done'
@six.add_metaclass(Singleton)
class StrategyStepNames(Constants):
@ -918,131 +923,13 @@ class SystemConfigUpdateHostsStep(strategy.StrategyStep):
return data
class UpgradeHostsStep(strategy.StrategyStep):
"""
Upgrade Hosts - Strategy Step
"""
def __init__(self, hosts):
super(UpgradeHostsStep, self).__init__(
STRATEGY_STEP_NAME.UPGRADE_HOSTS, timeout_in_secs=3600)
self._hosts = hosts
self._host_names = list()
self._host_uuids = list()
for host in hosts:
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
self._wait_time = 0
def _total_hosts_upgraded(self):
"""
Returns the number of hosts that are upgraded
"""
# TODO(jkraitbe): Use deploy/host_list instead
total_hosts_upgraded = 0
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is None:
return -1
if (host.is_online() and
host.target_load == self.strategy.nfvi_upgrade.release and
host.software_load == self.strategy.nfvi_upgrade.release):
total_hosts_upgraded += 1
return total_hosts_upgraded
def apply(self):
"""
Upgrade all hosts
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
self._host_names))
host_director = directors.get_host_director()
operation = host_director.upgrade_hosts(self._host_names)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_UPGRADE_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host upgrade failed")
return True
elif event in [STRATEGY_EVENT.HOST_AUDIT]:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) // 1000
# Wait at least 2 minutes for the host to go offline before
# checking whether the upgrade is complete.
if 120 <= secs_expired:
total_hosts_upgraded = self._total_hosts_upgraded()
if -1 == total_hosts_upgraded:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host no longer exists")
return True
if total_hosts_upgraded == len(self._host_names):
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
return True
return False
def from_dict(self, data):
"""
Returns the upgrade hosts step object initialized using the given
dictionary
"""
super(UpgradeHostsStep, self).from_dict(data)
self._hosts = list()
self._host_uuids = list()
self._host_names = data['entity_names']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
self._wait_time = 0
return self
def as_dict(self):
"""
Represent the upgrade hosts step as a dictionary
"""
data = super(UpgradeHostsStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
return data
class SwDeployPrecheckStep(strategy.StrategyStep):
"""
Software Deploy Precheck - Strategy Step
"""
def __init__(self, release):
super(SwDeployPrecheckStep, self).__init__(
STRATEGY_STEP_NAME.SW_DEPLOY_PRECHECK, timeout_in_secs=600)
STRATEGY_STEP_NAME.SW_DEPLOY_PRECHECK, timeout_in_secs=60)
self._release = release
@coroutine
@ -1054,12 +941,14 @@ class SwDeployPrecheckStep(strategy.StrategyStep):
DLOG.debug("sw-deploy precheck callback response=%s." % response)
if response['completed']:
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
# TODO(jkraitbe): Add error message
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
if not response['result-data']:
DLOG.debug("sw-deploy precheck completed %s" % response['result-data'])
else:
reason = "sw-deploy precheck failed"
# TODO(vselvara) to display the entire error response
# reason = response['result-data']
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, reason)
def apply(self):
"""
@ -1092,6 +981,120 @@ class SwDeployPrecheckStep(strategy.StrategyStep):
return data
class UpgradeHostsStep(strategy.StrategyStep):
"""
Upgrade Hosts - Strategy Step
"""
def __init__(self, hosts):
super(UpgradeHostsStep, self).__init__(
STRATEGY_STEP_NAME.UPGRADE_HOSTS, timeout_in_secs=3600)
self._host_names = list()
self._host_uuids = list()
for host in hosts:
self._host_names.append(host.name)
self._wait_time = 0
self._query_inprogress = False
@coroutine
def _get_upgrade_callback(self):
"""
Get Upgrade Callback
"""
response = (yield)
DLOG.debug("Query-Upgrade callback response=%s." % response)
self._query_inprogress = False
if response['completed']:
self.strategy.nfvi_upgrade = response['result-data']
host_count = 0
match_count = 0
host_info_list = self.strategy.nfvi_upgrade['hosts_info']
for host_name in self._host_names:
for host in host_info_list:
if (host_name == host['hostname']) and (host['host_state'] == SW_HOST_DEPLOYED):
match_count += 1
host_count += 1
if match_count == len(self._host_names):
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
DLOG.info("Upgrade Hosts completed")
self.stage.step_complete(result, "")
else:
# keep waiting for Upgrade host state to change
pass
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
DLOG.info("Host Upgrade failed")
self.stage.step_complete(result, response['reason'])
def apply(self):
"""
Upgrade all hosts
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
self._host_names))
host_director = directors.get_host_director()
operation = host_director.upgrade_hosts(self._host_names)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
from nfv_vim import nfvi
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_UPGRADE_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host upgrade failed")
return True
elif event in [STRATEGY_EVENT.HOST_AUDIT]:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) // 1000
# Wait at least 2 minutes for the host to go offline before
# checking whether the upgrade is complete.
if 120 <= secs_expired and not self._query_inprogress:
self._query_inprogress = True
release = self.strategy.nfvi_upgrade['release']
nfvi.nfvi_get_upgrade(release, self._get_upgrade_callback())
return True
return False
def from_dict(self, data):
"""
Returns the upgrade hosts step object initialized using the given
dictionary
"""
super(UpgradeHostsStep, self).from_dict(data)
self._host_uuids = list()
self._host_names = data['entity_names']
self._wait_time = 0
self._query_inprogress = False
return self
def as_dict(self):
"""
Represent the upgrade hosts step as a dictionary
"""
data = super(UpgradeHostsStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
return data
class UpgradeStartStep(strategy.StrategyStep):
"""
Upgrade Start - Strategy Step
@ -1109,15 +1112,19 @@ class UpgradeStartStep(strategy.StrategyStep):
"""
response = (yield)
DLOG.debug("Start-Upgrade callback response=%s." % response)
if response['completed']:
# TODO(jkraitbe): Consider updating self.strategy.nfvi_upgrade.hosts_info
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
state_info = response['result-data']['release_info']['state']
if state_info == SW_DEPLOY_START:
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
reason = "Software deploy not started. Current state:%s" % state_info
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, reason)
else:
# TODO(jkraitbe): Add error message
reason = "Software deploy start not completed"
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
self.stage.step_complete(result, reason)
def apply(self):
"""
@ -1167,15 +1174,20 @@ class UpgradeActivateStep(strategy.StrategyStep):
Activate Upgrade Callback
"""
response = (yield)
DLOG.debug("Activate-Upgrade callback response=%s." % response)
self.strategy.nfvi_upgrade = response['result-data']
if response['completed']:
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
state_info = self.strategy.nfvi_upgrade['release_info']['state']
if state_info == SW_DEPLOY_ACTIVATE_DONE:
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
reason = "SW Deploy Activate not successful. Current state:%s" % state_info
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, reason)
else:
# TODO(jkraitbe): Add error message
reason = "SW Deploy Activate not complete. Current response:%s" % response['result-data']
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
self.stage.step_complete(result, reason)
def apply(self):
"""
@ -1226,14 +1238,20 @@ class UpgradeCompleteStep(strategy.StrategyStep):
"""
response = (yield)
DLOG.debug("Complete-Upgrade callback response=%s." % response)
self.strategy.nfvi_upgrade = response['result-data']
if response['completed']:
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
state_info = self.strategy.nfvi_upgrade['release_info']['state']
if state_info == SW_HOST_DEPLOYED:
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
reason = "SW Deploy completed failed. Current state:%s" % state_info
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, reason)
else:
# TODO(jkraitbe): Add error message
reason = "SW Deploy did not complete. Current response:%s" % response['result-data']
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
self.stage.step_complete(result, reason)
def apply(self):
"""