openstack
/
fuel-web
Code Issues Proposed changes

Merge "Remove target images after env removal"

This commit is contained in:
Jenkins 2015-08-20 13:58:58 +00:00 committed by Gerrit Code Review
parent e9b04b47b0 0457984b5e
commit 9af57a0f31
5 changed files with 189 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
nailgun/nailgun/consts.py
View File

@ -206,6 +206,7 @@ TASK_NAMES = Enum(
'node_deletion',
'cluster_deletion',
'remove_images',
'check_before_deployment',
# network

1
nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_7_0.py
View File

@ -64,6 +64,7 @@ task_names_old = (
)
task_names_new = task_names_old + (
'spawn_vms',
'remove_images',
)

2
nailgun/nailgun/settings.yaml
View File

@ -97,6 +97,8 @@ REPO_PRIORITIES:
SHELL_TASK_RETRIES: 3
SHELL_TASK_INTERVAL: 1
REMOVE_IMAGES_TIMEOUT: 60
# statistics send interval
STATS_SEND_INTERVAL: 3600

48
nailgun/nailgun/task/task.py
View File

@ -16,6 +16,7 @@
import collections
from copy import deepcopy
import os
import netaddr
import requests
@ -43,6 +44,7 @@ from nailgun.orchestrator import deployment_graph
from nailgun.orchestrator import deployment_serializers
from nailgun.orchestrator import provisioning_serializers
from nailgun.orchestrator import stages
from nailgun.orchestrator import tasks_templates
from nailgun.settings import settings
from nailgun.task.fake import FAKE_THREADS
from nailgun.task.helpers import TaskHelper
@ -485,6 +487,41 @@ class DeletionTask(object):
return settings.FAKE_TASKS or settings.FAKE_TASKS_AMQP
class DeleteIBPImagesTask(object):
@classmethod
def message(cls, task, image_data):
files = []
for image in six.itervalues(image_data):
files.append(
os.path.join(
settings.PROVISIONING_IMAGES_PATH,
os.path.basename(
six.moves.urllib.parse.urlsplit(image['uri']).path))
)
task_params = {
'parameters': {
'cmd': 'rm -f {0}'.format(' '.join(files)),
'timeout': settings.REMOVE_IMAGES_TIMEOUT,
}
}
rpc_message = make_astute_message(
task,
'execute_tasks',
'remove_cluster_resp',
{
'tasks': [tasks_templates.make_shell_task([consts.MASTER_ROLE],
task_params),
]
}
)
return rpc_message
@classmethod
def execute(cls, task, image_data):
rpc.cast('naily', cls.message(task, image_data))
class StopDeploymentTask(object):
@classmethod
@ -581,6 +618,17 @@ class ClusterDeletionTask(object):
task,
nodes=DeletionTask.get_task_nodes_for_cluster(task.cluster),
respond_to='remove_cluster_resp')
attrs = objects.Attributes.merged_attrs_values(task.cluster.attributes)
if attrs.get('provision'):
if (task.cluster.release.operating_system ==
consts.RELEASE_OS.ubuntu and
attrs['provision']['method'] ==
consts.PROVISION_METHODS.image):
logger.debug("Delete IBP images task is running")
DeleteIBPImagesTask.execute(
task, attrs['provision']['image_data'])
else:
logger.debug("Skipping IBP images deletion task")
class BaseNetworkVerification(object):

178
nailgun/nailgun/test/unit/test_task.py
View File

@ -13,7 +13,8 @@
# License for the specific language governing permissions and limitations
# under the License.
from mock import patch
import mock
from oslo_serialization import jsonutils
import yaml
@ -22,11 +23,102 @@ from nailgun.db.sqlalchemy.models import Task
from nailgun.errors import errors
from nailgun.extensions.volume_manager.manager import VolumeManager
from nailgun import objects
from nailgun.task.task import CheckBeforeDeploymentTask
from nailgun.task import task
from nailgun.test.base import BaseTestCase
from nailgun.utils import reverse
class TestClusterDeletionTask(BaseTestCase):
def create_cluster_and_execute_deletion_task(
self, attributes=None, os=consts.RELEASE_OS.centos):
self.env.create(
cluster_kwargs={
'editable_attributes': attributes,
},
release_kwargs={
'operating_system': os,
'version': '2025-7.0',
},
)
self.fake_task = Task(name=consts.TASK_NAMES.cluster_deletion,
cluster=self.env.clusters[0])
task.ClusterDeletionTask.execute(self.fake_task)
@mock.patch('nailgun.task.task.DeletionTask', autospec=True)
@mock.patch.object(task.DeleteIBPImagesTask, 'execute')
def test_target_images_deletion_skipped_empty_attributes(
self, mock_img_task, mock_del):
self.create_cluster_and_execute_deletion_task({})
self.assertTrue(mock_del.execute.called)
self.assertFalse(mock_img_task.called)
@mock.patch('nailgun.task.task.DeletionTask', autospec=True)
@mock.patch.object(task.DeleteIBPImagesTask, 'execute')
def test_target_images_deletion_skipped_os_centos(
self, mock_img_task, mock_del):
attributes = {'provision': {
'method': consts.PROVISION_METHODS.image,
}}
self.create_cluster_and_execute_deletion_task(attributes)
self.assertTrue(mock_del.execute.called)
self.assertFalse(mock_img_task.called)
@mock.patch('nailgun.task.task.DeletionTask', autospec=True)
@mock.patch.object(task.DeleteIBPImagesTask, 'execute')
def test_target_images_deletion_skipped_os_ubuntu_cobbler(
self, mock_img_task, mock_del):
os = consts.RELEASE_OS.ubuntu
attributes = {'provision': {
'method': consts.PROVISION_METHODS.cobbler,
}}
self.create_cluster_and_execute_deletion_task(attributes, os)
self.assertTrue(mock_del.execute.called)
self.assertFalse(mock_img_task.called)
@mock.patch('nailgun.task.task.DeletionTask', autospec=True)
@mock.patch.object(task.DeleteIBPImagesTask, 'execute')
def test_target_images_deletion_executed(self, mock_img_task, mock_del):
os = consts.RELEASE_OS.ubuntu
attributes = {'provision': {
'method': consts.PROVISION_METHODS.image,
}}
self.create_cluster_and_execute_deletion_task(attributes, os)
self.assertTrue(mock_del.execute.called)
self.assertTrue(mock_img_task.called)
fake_attrs = objects.Attributes.merged_attrs_values(
self.fake_task.cluster.attributes)
mock_img_task.assert_called_once_with(
mock.ANY, fake_attrs['provision']['image_data'])
class TestDeleteIBPImagesTask(BaseTestCase):
@mock.patch('nailgun.task.task.settings')
@mock.patch('nailgun.task.task.make_astute_message')
def test_message(self, mock_astute, mock_settings):
mock_settings.PROVISIONING_IMAGES_PATH = '/fake/path'
mock_settings.REMOVE_IMAGES_TIMEOUT = 'fake_timeout'
task_mock = mock.Mock()
task_mock.cluster.id = '123'
task_mock.uuid = 'fake_uuid'
fake_image_data = {'/': {'uri': 'http://a.b/fake.img'},
'/boot': {'uri': 'http://c.d/fake-boot.img'}}
task.DeleteIBPImagesTask.message(task_mock, fake_image_data)
mock_astute.assert_called_once_with(
mock.ANY, 'execute_tasks', 'remove_cluster_resp', {
'tasks': [{
'type': 'shell',
'uids': [consts.MASTER_ROLE],
'parameters': {
'retries': 3,
'cmd': 'rm -f /fake/path/fake-boot.img '
'/fake/path/fake.img',
'cwd': '/',
'timeout': 'fake_timeout',
'interval': 1}}]})
class TestHelperUpdateClusterStatus(BaseTestCase):
def setUp(self):
@ -53,11 +145,12 @@ class TestHelperUpdateClusterStatus(BaseTestCase):
def test_update_nodes_to_error_if_deployment_task_failed(self):
self.cluster.nodes[0].status = 'deploying'
self.cluster.nodes[0].progress = 12
task = Task(name='deployment', cluster=self.cluster, status='error')
self.db.add(task)
deployment_task = Task(name='deployment', cluster=self.cluster,
status='error')
self.db.add(deployment_task)
self.db.commit()
objects.Task._update_cluster_data(task)
objects.Task._update_cluster_data(deployment_task)
self.db.flush()
self.assertEqual(self.cluster.status, 'error')
@ -65,11 +158,11 @@ class TestHelperUpdateClusterStatus(BaseTestCase):
self.nodes_should_not_be_error(self.cluster.nodes[1:])
def test_update_cluster_to_error_if_deploy_task_failed(self):
task = Task(name='deploy', cluster=self.cluster, status='error')
self.db.add(task)
deploy_task = Task(name='deploy', cluster=self.cluster, status='error')
self.db.add(deploy_task)
self.db.commit()
objects.Task._update_cluster_data(task)
objects.Task._update_cluster_data(deploy_task)
self.db.flush()
self.assertEqual(self.cluster.status, 'error')
@ -77,11 +170,12 @@ class TestHelperUpdateClusterStatus(BaseTestCase):
def test_update_nodes_to_error_if_provision_task_failed(self):
self.cluster.nodes[0].status = 'provisioning'
self.cluster.nodes[0].progress = 12
task = Task(name='provision', cluster=self.cluster, status='error')
self.db.add(task)
provision_task = Task(name='provision', cluster=self.cluster,
status='error')
self.db.add(provision_task)
self.db.commit()
objects.Task._update_cluster_data(task)
objects.Task._update_cluster_data(provision_task)
self.db.flush()
self.assertEqual(self.cluster.status, 'error')
@ -89,11 +183,11 @@ class TestHelperUpdateClusterStatus(BaseTestCase):
self.nodes_should_not_be_error(self.cluster.nodes[1:])
def test_update_cluster_to_operational(self):
task = Task(name='deploy', cluster=self.cluster, status='ready')
self.db.add(task)
deploy_task = Task(name='deploy', cluster=self.cluster, status='ready')
self.db.add(deploy_task)
self.db.commit()
objects.Task._update_cluster_data(task)
objects.Task._update_cluster_data(deploy_task)
self.db.flush()
self.assertEqual(self.cluster.status, 'operational')
@ -106,11 +200,11 @@ class TestHelperUpdateClusterStatus(BaseTestCase):
self.cluster.nodes[0].status = 'deploying'
self.cluster.nodes[0].progress = 24
task = Task(name='deploy', cluster=self.cluster, status='ready')
self.db.add(task)
deploy_task = Task(name='deploy', cluster=self.cluster, status='ready')
self.db.add(deploy_task)
self.db.commit()
objects.Task._update_cluster_data(task)
objects.Task._update_cluster_data(deploy_task)
self.db.flush()
self.assertEqual(self.cluster.status, 'operational')
@ -124,16 +218,17 @@ class TestHelperUpdateClusterStatus(BaseTestCase):
node.status = 'provisioning'
node.progress = 12
task = Task(name='provision', cluster=self.cluster, status='error')
self.db.add(task)
provision_task = Task(name='provision', cluster=self.cluster,
status='error')
self.db.add(provision_task)
self.db.commit()
data = {'status': 'error', 'progress': 100}
objects.Task.update(task, data)
objects.Task.update(provision_task, data)
self.db.flush()
self.assertEqual(self.cluster.status, 'error')
self.assertEqual(task.status, 'error')
self.assertEqual(provision_task.status, 'error')
for node in self.cluster.nodes:
self.assertEqual(node.status, 'error')
@ -190,7 +285,8 @@ class TestCheckBeforeDeploymentTask(BaseTestCase):
self.assertEqual(self.node.error_type, error_type)
def is_checking_required(self):
return CheckBeforeDeploymentTask._is_disk_checking_required(self.node)
return task.CheckBeforeDeploymentTask._is_disk_checking_required(
self.node)
def test_is_disk_checking_required(self):
self.set_node_status('ready')
@ -216,32 +312,32 @@ class TestCheckBeforeDeploymentTask(BaseTestCase):
def test_check_volumes_and_disks_do_not_run_if_node_ready(self):
self.set_node_status('ready')
with patch.object(
with mock.patch.object(
VolumeManager,
'check_disk_space_for_deployment') as check_mock:
CheckBeforeDeploymentTask._check_disks(self.task)
task.CheckBeforeDeploymentTask._check_disks(self.task)
self.assertFalse(check_mock.called)
with patch.object(
with mock.patch.object(
VolumeManager,
'check_volume_sizes_for_deployment') as check_mock:
CheckBeforeDeploymentTask._check_volumes(self.task)
task.CheckBeforeDeploymentTask._check_volumes(self.task)
self.assertFalse(check_mock.called)
def test_check_volumes_and_disks_run_if_node_not_ready(self):
self.set_node_status('discover')
with patch.object(
with mock.patch.object(
VolumeManager,
'check_disk_space_for_deployment') as check_mock:
CheckBeforeDeploymentTask._check_disks(self.task)
task.CheckBeforeDeploymentTask._check_disks(self.task)
self.assertEqual(check_mock.call_count, 1)
with patch.object(
with mock.patch.object(
VolumeManager,
'check_volume_sizes_for_deployment') as check_mock:
CheckBeforeDeploymentTask._check_volumes(self.task)
task.CheckBeforeDeploymentTask._check_volumes(self.task)
self.assertEqual(check_mock.call_count, 1)
@ -251,7 +347,7 @@ class TestCheckBeforeDeploymentTask(BaseTestCase):
self.assertRaises(
errors.NodeOffline,
CheckBeforeDeploymentTask._check_nodes_are_online,
task.CheckBeforeDeploymentTask._check_nodes_are_online,
self.task)
def test_check_nodes_online_do_not_raise_exception_node_to_deletion(self):
@ -259,7 +355,7 @@ class TestCheckBeforeDeploymentTask(BaseTestCase):
self.node.pending_deletion = True
self.env.db.commit()
CheckBeforeDeploymentTask._check_nodes_are_online(self.task)
task.CheckBeforeDeploymentTask._check_nodes_are_online(self.task)
def test_check_controllers_count_operational_cluster(self):
self.cluster.status = consts.CLUSTER_STATUSES.operational
@ -271,7 +367,7 @@ class TestCheckBeforeDeploymentTask(BaseTestCase):
self.assertRaises(
errors.NotEnoughControllers,
CheckBeforeDeploymentTask._check_controllers_count,
task.CheckBeforeDeploymentTask._check_controllers_count,
self.task)
def test_check_controllers_count_new_cluster(self):
@ -280,7 +376,7 @@ class TestCheckBeforeDeploymentTask(BaseTestCase):
# check there's not exceptions with one controller
self.assertNotRaises(
errors.NotEnoughControllers,
CheckBeforeDeploymentTask._check_controllers_count,
task.CheckBeforeDeploymentTask._check_controllers_count,
self.task)
# check there's exception with one non-controller node
@ -288,7 +384,7 @@ class TestCheckBeforeDeploymentTask(BaseTestCase):
self.env.db.flush()
self.assertRaises(
errors.NotEnoughControllers,
CheckBeforeDeploymentTask._check_controllers_count,
task.CheckBeforeDeploymentTask._check_controllers_count,
self.task)
def find_net_by_name(self, nets, name):
@ -305,14 +401,14 @@ class TestCheckBeforeDeploymentTask(BaseTestCase):
self.assertNotRaises(
errors.NetworktemplateMissingRoles,
CheckBeforeDeploymentTask._validate_network_template,
task.CheckBeforeDeploymentTask._validate_network_template,
self.task)
ceph_node = self.env.create_node(roles=['ceph-osd'],
cluster_id=self.cluster.id)
self.assertRaises(
errors.NetworkTemplateMissingRoles,
CheckBeforeDeploymentTask._validate_network_template,
task.CheckBeforeDeploymentTask._validate_network_template,
self.task)
objects.Node.delete(ceph_node)
@ -326,7 +422,7 @@ class TestCheckBeforeDeploymentTask(BaseTestCase):
self.assertRaisesRegexp(
errors.NetworkTemplateMissingNetRoles,
"Network roles murano/api are missing",
CheckBeforeDeploymentTask._validate_network_template,
task.CheckBeforeDeploymentTask._validate_network_template,
self.task)
def test_check_public_networks(self):
@ -360,7 +456,7 @@ class TestCheckBeforeDeploymentTask(BaseTestCase):
self.assertRaises(
errors.NetworkCheckError,
CheckBeforeDeploymentTask._check_public_network,
task.CheckBeforeDeploymentTask._check_public_network,
self.task)
# enough IPs for 3 nodes and 2 VIPs
@ -371,7 +467,7 @@ class TestCheckBeforeDeploymentTask(BaseTestCase):
self.assertNotRaises(
errors.NetworkCheckError,
CheckBeforeDeploymentTask._check_public_network,
task.CheckBeforeDeploymentTask._check_public_network,
self.task)
attrs['public_network_assignment']['assign_to_all_nodes']['value'] = \
@ -389,7 +485,7 @@ class TestCheckBeforeDeploymentTask(BaseTestCase):
self.assertRaises(
errors.NetworkCheckError,
CheckBeforeDeploymentTask._check_public_network,
task.CheckBeforeDeploymentTask._check_public_network,
self.task)
def test_check_deployment_graph_with_correct_data(self):