From a18bdbf92b5c7d5dcb53e52f6ec97d39df7254d3 Mon Sep 17 00:00:00 2001 From: xiaodongwang Date: Tue, 28 Apr 2015 14:33:30 -0700 Subject: [PATCH] update cluster state to error when deployment fails Change-Id: I54fb25d6151679118cf648b5c2192d56f7d95957 --- bin/refresh.sh | 16 ++++ compass/actions/deploy.py | 50 +++++++++--- compass/actions/util.py | 8 +- compass/db/api/cluster.py | 3 +- compass/db/models.py | 71 ++++++++--------- .../cobbler/CentOS-6.5-x86_64/system.tmpl | 8 +- .../cobbler/CentOS-6.6-x86_64/system.tmpl | 8 +- .../cobbler/CentOS-7.0-x86_64/system.tmpl | 8 +- .../cobbler/Ubuntu-12.04-x86_64/system.tmpl | 8 +- .../cobbler/Ubuntu-14.04-x86_64/system.tmpl | 8 +- .../cobbler/sles-11sp3-x86_64/system.tmpl | 8 +- .../cobbler/uvp-11sp3-x86_64/system.tmpl | 76 +++++++++++++++++++ install/compass.sh | 3 - 13 files changed, 216 insertions(+), 59 deletions(-) create mode 100644 conf/templates/cobbler/uvp-11sp3-x86_64/system.tmpl diff --git a/bin/refresh.sh b/bin/refresh.sh index fd553b43..ddad90f0 100755 --- a/bin/refresh.sh +++ b/bin/refresh.sh @@ -1,15 +1,31 @@ #!/bin/bash set -e service mysqld restart +service mysqld status || exit $? /opt/compass/bin/manage_db.py createdb /opt/compass/bin/clean_installers.py --noasync /opt/compass/bin/clean_installation_logs.py rm -rf /var/ansible/run/* service httpd restart +sleep 10 +service httpd status || exit $? service rsyslog restart +sleep 10 +service rsyslog status || exit $? service redis restart sleep 10 +service redis status || exit $? redis-cli flushall +service cobblerd restart +sleep 10 +service cobblerd status || exit $? +chef-server-ctl restart +sleep 10 +chef-server-ctl status || exit $? service compass-celeryd restart +sleep 10 +service compass-celeryd status || exit $? service compass-progress-updated restart +sleep 10 +service compass-progress-updated status || exit $? diff --git a/compass/actions/deploy.py b/compass/actions/deploy.py index a4c5a6a0..f8639c0b 100644 --- a/compass/actions/deploy.py +++ b/compass/actions/deploy.py @@ -46,14 +46,28 @@ def deploy(cluster_id, hosts_id_list, username=None): hosts_info = util.ActionHelper.get_hosts_info( cluster_id, hosts_id_list, user) - deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) - # deploy_manager.prepare_for_deploy() - logging.debug('Created deploy manager with %s %s %s' - % (adapter_info, cluster_info, hosts_info)) + deploy_successful = True + try: + deploy_manager = DeployManager( + adapter_info, cluster_info, hosts_info) + # deploy_manager.prepare_for_deploy() + logging.debug('Created deploy manager with %s %s %s' + % (adapter_info, cluster_info, hosts_info)) + deployed_config = deploy_manager.deploy() + except Exception as error: + logging.exception(error) + deploy_successful = False - deployed_config = deploy_manager.deploy() - util.ActionHelper.save_deployed_config(deployed_config, user) - util.ActionHelper.update_state(cluster_id, hosts_id_list, user) + if deploy_successful: + util.ActionHelper.save_deployed_config(deployed_config, user) + util.ActionHelper.update_state( + cluster_id, hosts_id_list, user, state='INSTALLING' + ) + else: + util.ActionHelper.update_state( + cluster_id, hosts_id_list, user, state='ERROR', + message='failed to start deployment', severity='ERROR' + ) def redeploy(cluster_id, hosts_id_list, username=None): @@ -75,10 +89,24 @@ def redeploy(cluster_id, hosts_id_list, username=None): hosts_info = util.ActionHelper.get_hosts_info( cluster_id, hosts_id_list, user) - deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) - # deploy_manager.prepare_for_deploy() - deploy_manager.redeploy() - util.ActionHelper.update_state(cluster_id, hosts_id_list, user) + deploy_successful = True + try: + deploy_manager = DeployManager( + adapter_info, cluster_info, hosts_info) + # deploy_manager.prepare_for_deploy() + deploy_manager.redeploy() + except Exception as error: + logging.exception(error) + deploy_successful = False + if deploy_successful: + util.ActionHelper.update_state( + cluster_id, hosts_id_list, user, state='INSTALLING', + ) + else: + util.ActionHelper.update_state( + cluster_id, hosts_id_list, user, state='ERROR', + message='failed to start redeployment', severity='ERROR' + ) def health_check(cluster_id, report_uri, username): diff --git a/compass/actions/util.py b/compass/actions/util.py index 3a9b4b3b..9aba09a0 100644 --- a/compass/actions/util.py +++ b/compass/actions/util.py @@ -232,21 +232,23 @@ class ActionHelper(object): **config) @staticmethod - def update_state(cluster_id, host_id_list, user): + def update_state( + cluster_id, host_id_list, user, **kwargs + ): # update all clusterhosts state for host_id in host_id_list: cluster_db.update_cluster_host_state( cluster_id, host_id, user=user, - state='INSTALLING' + **kwargs ) # update cluster state cluster_db.update_cluster_state( cluster_id, user=user, - state='INSTALLING' + **kwargs ) @staticmethod diff --git a/compass/db/api/cluster.py b/compass/db/api/cluster.py index d377d6e3..45ee8066 100644 --- a/compass/db/api/cluster.py +++ b/compass/db/api/cluster.py @@ -134,6 +134,7 @@ UPDATED_CLUSTERHOST_STATE_INTERNAL_FIELDS = [ 'ready' ] UPDATED_CLUSTER_STATE_FIELDS = ['state'] +IGNORE_UPDATED_CLUSTER_STATE_FIELDS = ['percentage', 'message', 'severity'] UPDATED_CLUSTER_STATE_INTERNAL_FIELDS = ['ready'] RESP_CLUSTERHOST_LOG_FIELDS = [ 'clusterhost_id', 'id', 'host_id', 'cluster_id', @@ -1916,7 +1917,7 @@ def update_clusterhost_state_internal( @utils.supported_filters( optional_support_keys=UPDATED_CLUSTER_STATE_FIELDS, - ignore_support_keys=IGNORE_FIELDS + ignore_support_keys=(IGNORE_FIELDS + IGNORE_UPDATED_CLUSTER_STATE_FIELDS) ) @database.run_in_session() @user_api.check_user_permission_in_session( diff --git a/compass/db/models.py b/compass/db/models.py index d27acaaf..b272a8f0 100644 --- a/compass/db/models.py +++ b/compass/db/models.py @@ -1152,42 +1152,43 @@ class ClusterState(BASE, StateMixin): self.completed_hosts = 0 if self.state == 'INSTALLING': cluster.reinstall_distributed_system = False - if not cluster.distributed_system: - for clusterhost in clusterhosts: - host = clusterhost.host - host_state = host.state.state - if host_state == 'INSTALLING': - self.installing_hosts += 1 - elif host_state == 'ERROR': - self.failed_hosts += 1 - elif host_state == 'SUCCESSFUL': - self.completed_hosts += 1 + + if not cluster.distributed_system: + for clusterhost in clusterhosts: + host = clusterhost.host + host_state = host.state.state + if host_state == 'INSTALLING': + self.installing_hosts += 1 + elif host_state == 'ERROR': + self.failed_hosts += 1 + elif host_state == 'SUCCESSFUL': + self.completed_hosts += 1 + else: + for clusterhost in clusterhosts: + clusterhost_state = clusterhost.state.state + if clusterhost_state == 'INSTALLING': + self.installing_hosts += 1 + elif clusterhost_state == 'ERROR': + self.failed_hosts += 1 + elif clusterhost_state == 'SUCCESSFUL': + self.completed_hosts += 1 + if self.total_hosts: + if self.completed_hosts == self.total_hosts: + self.percentage = 1.0 else: - for clusterhost in clusterhosts: - clusterhost_state = clusterhost.state.state - if clusterhost_state == 'INSTALLING': - self.installing_hosts += 1 - elif clusterhost_state == 'ERROR': - self.failed_hosts += 1 - elif clusterhost_state == 'SUCCESSFUL': - self.completed_hosts += 1 - if self.total_hosts: - if self.completed_hosts == self.total_hosts: - self.percentage = 1.0 - else: - self.percentage = ( - float(self.completed_hosts) - / - float(self.total_hosts) - ) - self.message = ( - 'total %s, installing %s, completed: %s, error %s' - ) % ( - self.total_hosts, self.installing_hosts, - self.completed_hosts, self.failed_hosts - ) - if self.failed_hosts: - self.severity = 'ERROR' + self.percentage = ( + float(self.completed_hosts) + / + float(self.total_hosts) + ) + self.message = ( + 'total %s, installing %s, completed: %s, error %s' + ) % ( + self.total_hosts, self.installing_hosts, + self.completed_hosts, self.failed_hosts + ) + if self.failed_hosts: + self.severity = 'ERROR' super(ClusterState, self).update() diff --git a/conf/templates/cobbler/CentOS-6.5-x86_64/system.tmpl b/conf/templates/cobbler/CentOS-6.5-x86_64/system.tmpl index 34c9c5e3..8e4cbbbe 100644 --- a/conf/templates/cobbler/CentOS-6.5-x86_64/system.tmpl +++ b/conf/templates/cobbler/CentOS-6.5-x86_64/system.tmpl @@ -27,7 +27,13 @@ #if $val.is_mgmt #set $rekeyed[$dns_key] = $dns #else - #set $rekeyed[$dns_key] = '-'.join(($dns, $nic)) + #if '.' in $dns + #set $dns_name, $dns_domain = $dns.split('.', 1) + #set $dns_nic = '%s-%s.%s' % ($dns_name, $nic, $dns_domain) + #else + #set $dns_nic = '%s-%s' % ($dns, $nic) + #end if + #set $rekeyed[$dns_key] = $dns_nic #end if #if $val.is_promiscuous: diff --git a/conf/templates/cobbler/CentOS-6.6-x86_64/system.tmpl b/conf/templates/cobbler/CentOS-6.6-x86_64/system.tmpl index 34c9c5e3..8e4cbbbe 100644 --- a/conf/templates/cobbler/CentOS-6.6-x86_64/system.tmpl +++ b/conf/templates/cobbler/CentOS-6.6-x86_64/system.tmpl @@ -27,7 +27,13 @@ #if $val.is_mgmt #set $rekeyed[$dns_key] = $dns #else - #set $rekeyed[$dns_key] = '-'.join(($dns, $nic)) + #if '.' in $dns + #set $dns_name, $dns_domain = $dns.split('.', 1) + #set $dns_nic = '%s-%s.%s' % ($dns_name, $nic, $dns_domain) + #else + #set $dns_nic = '%s-%s' % ($dns, $nic) + #end if + #set $rekeyed[$dns_key] = $dns_nic #end if #if $val.is_promiscuous: diff --git a/conf/templates/cobbler/CentOS-7.0-x86_64/system.tmpl b/conf/templates/cobbler/CentOS-7.0-x86_64/system.tmpl index 34c9c5e3..8e4cbbbe 100644 --- a/conf/templates/cobbler/CentOS-7.0-x86_64/system.tmpl +++ b/conf/templates/cobbler/CentOS-7.0-x86_64/system.tmpl @@ -27,7 +27,13 @@ #if $val.is_mgmt #set $rekeyed[$dns_key] = $dns #else - #set $rekeyed[$dns_key] = '-'.join(($dns, $nic)) + #if '.' in $dns + #set $dns_name, $dns_domain = $dns.split('.', 1) + #set $dns_nic = '%s-%s.%s' % ($dns_name, $nic, $dns_domain) + #else + #set $dns_nic = '%s-%s' % ($dns, $nic) + #end if + #set $rekeyed[$dns_key] = $dns_nic #end if #if $val.is_promiscuous: diff --git a/conf/templates/cobbler/Ubuntu-12.04-x86_64/system.tmpl b/conf/templates/cobbler/Ubuntu-12.04-x86_64/system.tmpl index 2d7f1432..cfcc883e 100644 --- a/conf/templates/cobbler/Ubuntu-12.04-x86_64/system.tmpl +++ b/conf/templates/cobbler/Ubuntu-12.04-x86_64/system.tmpl @@ -26,7 +26,13 @@ #if $val.is_mgmt #set $rekeyed[$dns_key] = $dns #else - #set $rekeyed[$dns_key] = '-'.join(($dns, $nic)) + #if '.' in $dns + #set $dns_name, $dns_domain = $dns.split('.', 1) + #set $dns_nic = '%s-%s.%s' % ($dns_name, $nic, $dns_domain) + #else + #set $dns_nic = '%s-%s' % ($dns, $nic) + #end if + #set $rekeyed[$dns_key] = $dns_nic #end if #if $val.is_promiscuous: diff --git a/conf/templates/cobbler/Ubuntu-14.04-x86_64/system.tmpl b/conf/templates/cobbler/Ubuntu-14.04-x86_64/system.tmpl index 2d7f1432..cfcc883e 100644 --- a/conf/templates/cobbler/Ubuntu-14.04-x86_64/system.tmpl +++ b/conf/templates/cobbler/Ubuntu-14.04-x86_64/system.tmpl @@ -26,7 +26,13 @@ #if $val.is_mgmt #set $rekeyed[$dns_key] = $dns #else - #set $rekeyed[$dns_key] = '-'.join(($dns, $nic)) + #if '.' in $dns + #set $dns_name, $dns_domain = $dns.split('.', 1) + #set $dns_nic = '%s-%s.%s' % ($dns_name, $nic, $dns_domain) + #else + #set $dns_nic = '%s-%s' % ($dns, $nic) + #end if + #set $rekeyed[$dns_key] = $dns_nic #end if #if $val.is_promiscuous: diff --git a/conf/templates/cobbler/sles-11sp3-x86_64/system.tmpl b/conf/templates/cobbler/sles-11sp3-x86_64/system.tmpl index 34c9c5e3..8e4cbbbe 100644 --- a/conf/templates/cobbler/sles-11sp3-x86_64/system.tmpl +++ b/conf/templates/cobbler/sles-11sp3-x86_64/system.tmpl @@ -27,7 +27,13 @@ #if $val.is_mgmt #set $rekeyed[$dns_key] = $dns #else - #set $rekeyed[$dns_key] = '-'.join(($dns, $nic)) + #if '.' in $dns + #set $dns_name, $dns_domain = $dns.split('.', 1) + #set $dns_nic = '%s-%s.%s' % ($dns_name, $nic, $dns_domain) + #else + #set $dns_nic = '%s-%s' % ($dns, $nic) + #end if + #set $rekeyed[$dns_key] = $dns_nic #end if #if $val.is_promiscuous: diff --git a/conf/templates/cobbler/uvp-11sp3-x86_64/system.tmpl b/conf/templates/cobbler/uvp-11sp3-x86_64/system.tmpl new file mode 100644 index 00000000..8e4cbbbe --- /dev/null +++ b/conf/templates/cobbler/uvp-11sp3-x86_64/system.tmpl @@ -0,0 +1,76 @@ +{ + "name": "$hostname", + "hostname": "$hostname", + "profile": "$profile", + "gateway": "$gateway", + #import simplejson as json + #set nameservers = json.dumps($nameservers, encoding='utf-8') + "name_servers": $nameservers, + #set search_path = ' '.join($search_path) + "name_servers_search": "$search_path", + "proxy": "$getVar('http_proxy', '')", + "modify_interface": + #set networks = $networks + #set rekeyed = {} + #set promic_nic = "" + #for $nic, $val in $networks.iteritems(): + #set ip_key = '-'.join(('ipaddress', $nic)) + #set netmask_key = '-'.join(('netmask', $nic)) + #set mgmt_key = '-'.join(('management', $nic)) + #set static_key = '-'.join(('static', $nic)) + #set $rekeyed[$ip_key] = $val.ip + #set $rekeyed[$netmask_key] = $val.netmask + #set $rekeyed[$mgmt_key] = $val.is_mgmt + #set $rekeyed[$static_key] = True + + #set dns_key = '-'.join(('dnsname', $nic)) + #if $val.is_mgmt + #set $rekeyed[$dns_key] = $dns + #else + #if '.' in $dns + #set $dns_name, $dns_domain = $dns.split('.', 1) + #set $dns_nic = '%s-%s.%s' % ($dns_name, $nic, $dns_domain) + #else + #set $dns_nic = '%s-%s' % ($dns, $nic) + #end if + #set $rekeyed[$dns_key] = $dns_nic + #end if + + #if $val.is_promiscuous: + #set promic_nic = $nic + #end if + #if $val.is_mgmt: + #set mac_key = '-'.join(('macaddress', $nic)) + #set $rekeyed[$mac_key] = $mac + #end if + #end for + #set nic_info = json.dumps($rekeyed, encoding='utf-8') + $nic_info + , + "ksmeta":{ + #set partition_config = '' + #for k, v in $partition.iteritems(): + #set path = '' + #if v['vol_percentage']: + #set $path = k + ' ' + str(v['vol_percentage']) + '%' + #else: + #set $path = k + ' ' + str(v['vol_size']) + #end if + #set partition_config = ';'.join((partition_config, $path)) + #end for + #set partition_config = partition_config[1:] + #import crypt + #set $password = crypt.crypt($server_credentials.password, "az") + #set no_proxy = ','.join($getVar('no_proxy', [])) + "username": "$server_credentials.username", + "password": "$password", + "promisc_nics": "$promic_nic", + "partition": "$partition_config", + "https_proxy": "$getVar('https_proxy', '')", + "ntp_server": "$ntp_server", + "timezone": "$timezone", + "ignore_proxy": "$no_proxy", + "local_repo": "$getVar('local_repo', '')", + "disk_num": "1" + } +} diff --git a/install/compass.sh b/install/compass.sh index 8f06b307..ccbb8f88 100755 --- a/install/compass.sh +++ b/install/compass.sh @@ -163,9 +163,6 @@ if [[ "$?" != "0" ]]; then exit 1 fi -sudo killall -9 celery -sudo service compass-celeryd restart -sudo sleep 10 sudo service compass-celeryd status |grep running if [[ "$?" != "0" ]]; then echo "compass-celeryd is not started"