openstack
/
fuel-web
Code Issues Proposed changes

Removing keystone user on the customer stats opt-out 

For removing keystone user on customer stats opt-out we are using
astute 'execute_task' with type 'puppet'. Puppet granule
'remove_workloads_collector.pp' used for it. Puppet manifest will
be applied only on primary-controllers of operational clusters.

Items of the change:

- Task name 'remove_stats_user' added
- Task RemoveStatsUserTask for executing puppet manifest for
  removing stats user added
- Logging of the task RemovedStatsUserTask into action_logs handled
- Base class for stats task managers extracted
- RPC method for handling stats user operations was generalized

Change-Id: I07906798fe051efb7ea88228f5953b87ddc618a8
Depends-On: I377a32e75396269ee8390a33ebfd4c110036ffca
Closes-Bug: #1454561
This commit is contained in:
Alexander Kislitsky 2015-05-13 13:28:13 +03:00
parent d14f1e508c
commit 164b584662
10 changed files with 229 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
nailgun/nailgun/api/v1/handlers/master_node_settings.py
View File

@ -19,6 +19,7 @@ from nailgun.api.v1.validators.master_node_settings \
from nailgun.logger import logger
from nailgun import objects
from nailgun.task.manager import CreateStatsUserTaskManager
from nailgun.task.manager import RemoveStatsUserTaskManager
class MasterNodeSettingsHandler(DBSingletonHandler):
@ -30,13 +31,18 @@ class MasterNodeSettingsHandler(DBSingletonHandler):
not_found_error = "Settings are not found in DB"
def _handle_stats_opt_in(self):
if self.single.must_send_stats():
logger.debug("Handling customer opt-in to sending statistics")
try:
manager = CreateStatsUserTaskManager()
manager.execute()
except Exception:
logger.exception("Creating stats user failed")
manager = CreateStatsUserTaskManager()
else:
logger.debug("Handling customer opt-out to sending statistics")
manager = RemoveStatsUserTaskManager()
try:
manager.execute()
except Exception:
logger.exception("Stats user operation failed")
@content
def PUT(self):

3
nailgun/nailgun/consts.py
View File

@ -220,7 +220,8 @@ TASK_NAMES = Enum(
'capacity_log',
# statistics
'create_stats_user'
'create_stats_user',
'remove_stats_user'
)
NOTIFICATION_STATUSES = Enum(

3
nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_6_1.py
View File

@ -135,7 +135,8 @@ task_names_old = (
'dump',
'capacity_log'
)
task_names_new = task_names_old + ('create_stats_user',)
task_names_new = task_names_old + ('create_stats_user',
'remove_stats_user')
def upgrade():

13
nailgun/nailgun/rpc/receiver.py
View File

@ -1091,8 +1091,8 @@ class NailgunReceiver(object):
objects.Task.update(task, data)
@classmethod
def create_stats_user_resp(cls, **kwargs):
logger.info("RPC method create_stats_user_resp received: %s",
def stats_user_resp(cls, **kwargs):
logger.info("RPC method stats_user_resp received: %s",
jsonutils.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
@ -1106,15 +1106,16 @@ class NailgunReceiver(object):
if status not in (consts.TASK_STATUSES.ready,
consts.TASK_STATUSES.error):
logger.debug("Creating stats user task %s in status: %s",
task.id, task.status)
logger.debug("Task %s, id: %s in status: %s",
task.name, task.id, task.status)
return
data = {'status': status, 'progress': 100, 'message': message}
if status == consts.TASK_STATUSES.error:
logger.error("Creating status user failed: %s", error)
logger.error("Task %s, id: %s failed: %s",
task.name, task.id, error)
data['message'] = error
objects.Task.update(task, data)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
logger.info("RPC method create_stats_user_resp processed")
logger.info("RPC method stats_user_resp processed")

3
nailgun/nailgun/task/helpers.py
View File

@ -50,7 +50,8 @@ tasks_names_actions_groups_mapping = {
consts.TASK_NAMES.dump: "operations",
consts.TASK_NAMES.capacity_log: "operations",
consts.TASK_NAMES.create_stats_user: "statistics"
consts.TASK_NAMES.create_stats_user: "statistics",
consts.TASK_NAMES.remove_stats_user: "statistics"
}

44
nailgun/nailgun/task/manager.py
View File

@ -1093,48 +1093,64 @@ class NodeDeletionTaskManager(TaskManager):
return task
class CreateStatsUserTaskManager(TaskManager):
class BaseStatsUserTaskManager(TaskManager):
task_name = None
task_cls = None
def execute(self):
logger.info("Trying to create stats users in the operational "
"environments")
logger.info("Trying to execute %s in the operational "
"environments", self.task_name)
created_tasks = []
clusters = objects.ClusterCollection.filter_by(
None, status=consts.CLUSTER_STATUSES.operational)
for cluster in clusters:
logger.debug("Creating task for adding stats user on the "
"cluster %s", cluster.id)
logger.debug("Creating task for %s on the "
"cluster %s", self.task_name, cluster.id)
primary_controller = objects.Cluster.get_primary_node(
cluster, 'controller')
if primary_controller is not None:
logger.debug("Primary controller cluster %s found: %s. "
"Creating task for creating fuel stats user",
cluster.id, primary_controller.id)
"Creating task for %s", cluster.id,
primary_controller.id, self.task_name)
if objects.TaskCollection.filter_by(
None,
cluster_id=cluster.id,
name=consts.TASK_NAMES.create_stats_user,
name=self.task_name,
status=consts.TASK_STATUSES.running).count():
logger.debug("Creating stats user task is already "
"running for cluster %s", cluster.id)
logger.debug("Task %s is already running for cluster %s",
self.task_name, cluster.id)
continue
task = Task(name=consts.TASK_NAMES.create_stats_user,
cluster_id=cluster.id)
task = Task(name=self.task_name, cluster_id=cluster.id)
db().add(task)
db().commit()
created_tasks.append(task)
self._call_silently(
task,
tasks.CreateStatsUserTask,
self.task_cls,
primary_controller
)
else:
logger.debug("Primary controller not found for cluster %s",
cluster.id)
return created_tasks
class CreateStatsUserTaskManager(BaseStatsUserTaskManager):
task_name = consts.TASK_NAMES.create_stats_user
task_cls = tasks.CreateStatsUserTask
class RemoveStatsUserTaskManager(BaseStatsUserTaskManager):
task_name = consts.TASK_NAMES.remove_stats_user
task_cls = tasks.RemoveStatsUserTask

34
nailgun/nailgun/task/task.py
View File

@ -1158,7 +1158,7 @@ class CreateStatsUserTask(object):
rpc_message = make_astute_message(
task,
'execute_tasks',
'create_stats_user_resp',
'stats_user_resp',
{
'tasks': [{
'type': consts.ORCHESTRATOR_TASK_TYPES.puppet,
@ -1183,6 +1183,38 @@ class CreateStatsUserTask(object):
)
class RemoveStatsUserTask(object):
@classmethod
def message(cls, task, primary_controller):
rpc_message = make_astute_message(
task,
'execute_tasks',
'stats_user_resp',
{
'tasks': [{
'type': consts.ORCHESTRATOR_TASK_TYPES.puppet,
'uids': [primary_controller.id],
'parameters': {
'puppet_modules': '/etc/puppet/modules',
'puppet_manifest': '/etc/puppet/modules/osnailyfacter'
'/modular/keystone'
'/remove_workloads_collector.pp',
'cwd': '/',
}
}]
}
)
return rpc_message
@classmethod
def execute(cls, task, primary_controller):
rpc.cast(
'naily',
cls.message(task, primary_controller)
)
if settings.FAKE_TASKS or settings.FAKE_TASKS_AMQP:
rpc.cast = fake_cast
CheckRepositoryConnectionTask._get_failed_repositories = classmethod(

29
nailgun/nailgun/test/integration/test_action_logs.py
View File

@ -129,3 +129,32 @@ class TestActionLogs(BaseMasterNodeSettignsTest):
action_log = objects.ActionLogCollection.filter_by(
None, task_uuid=task.uuid)
self.assertIsNotNone(action_log)
@fake_tasks(override_state={'progress': 100,
'status': consts.TASK_STATUSES.ready})
def test_remove_stats_user_logged(self):
self.env.create(
nodes_kwargs=[
{'roles': ['controller'], 'pending_addition': True},
]
)
with mock.patch('nailgun.objects.MasterNodeSettings.must_send_stats',
return_value=True):
deploy_task = self.env.launch_deployment()
self.env.wait_ready(deploy_task)
with mock.patch('nailgun.objects.MasterNodeSettings.must_send_stats',
return_value=False):
resp = self.app.patch(
reverse('MasterNodeSettingsHandler'),
headers=self.default_headers,
params='{}'
)
self.assertEqual(200, resp.status_code)
task = objects.TaskCollection.filter_by(
None, name=consts.TASK_NAMES.remove_stats_user).first()
action_log = objects.ActionLogCollection.filter_by(
None, task_uuid=task.uuid)
self.assertIsNotNone(action_log)

4
nailgun/nailgun/test/integration/test_orchestrator_serializer.py
View File

@ -2535,7 +2535,9 @@ class TestDeploymentAttributesSerialization61(BaseDeploymentSerializer):
objects.NodeCollection.prepare_for_deployment(self.env.nodes, 'gre')
self.serializer = DeploymentHASerializer61(self.cluster)
def test_serialize_workloads_collector_user(self):
@mock.patch('nailgun.objects.MasterNodeSettings.must_send_stats',
return_value=False)
def test_serialize_workloads_collector_user_opted_out(self, _):
oswl_user = self.serializer.get_common_attrs(
self.env.clusters[0]
)['workloads_collector']

147
nailgun/nailgun/test/integration/test_stats_user_task_manager.py
View File

@ -66,30 +66,38 @@ class TestStatsUserTaskManagers(BaseMasterNodeSettignsTest):
deploy_task = self.env.launch_deployment()
self.env.wait_ready(deploy_task)
task_count_before = objects.TaskCollection.filter_by(
None, name=consts.TASK_NAMES.create_stats_user).count()
# Tuple of tuples (task_name, must_send_stats)
tasks_params = (
(consts.TASK_NAMES.create_stats_user, True),
(consts.TASK_NAMES.remove_stats_user, False)
)
with mock.patch('nailgun.objects.MasterNodeSettings.must_send_stats',
return_value=True):
with mock.patch('nailgun.task.fake.settings.'
'FAKE_TASKS_TICK_INTERVAL', 10):
resp = self.app.patch(
reverse('MasterNodeSettingsHandler'),
headers=self.default_headers,
params='{}'
)
self.assertEqual(200, resp.status_code)
for task_name, must_send_stats in tasks_params:
resp = self.app.patch(
reverse('MasterNodeSettingsHandler'),
headers=self.default_headers,
params='{}'
)
self.assertEqual(200, resp.status_code)
task_count_before = objects.TaskCollection.filter_by(
None, name=task_name).count()
task_count = objects.TaskCollection.filter_by(
None, name=consts.TASK_NAMES.create_stats_user).count()
self.assertEqual(task_count_before + 1, task_count)
with mock.patch('nailgun.objects.MasterNodeSettings.'
'must_send_stats', return_value=must_send_stats):
with mock.patch('nailgun.task.fake.settings.'
'FAKE_TASKS_TICK_INTERVAL', 10):
resp = self.app.patch(
reverse('MasterNodeSettingsHandler'),
headers=self.default_headers,
params='{}'
)
self.assertEqual(200, resp.status_code)
resp = self.app.patch(
reverse('MasterNodeSettingsHandler'),
headers=self.default_headers,
params='{}'
)
self.assertEqual(200, resp.status_code)
task_count = objects.TaskCollection.filter_by(
None, name=task_name).count()
self.assertEqual(task_count_before + 1, task_count)
@fake_tasks(override_state={'progress': 100,
'status': consts.TASK_STATUSES.ready})
@ -104,25 +112,35 @@ class TestStatsUserTaskManagers(BaseMasterNodeSettignsTest):
self.env.wait_ready(deploy_task)
cluster = self.env.clusters[0]
with mock.patch('nailgun.objects.MasterNodeSettings.must_send_stats',
return_value=True):
for cluster_status in consts.CLUSTER_STATUSES:
if cluster_status == consts.CLUSTER_STATUSES.operational:
continue
cluster.status = cluster_status
self.env.db().flush()
# Tuple of tuples (task_name, must_send_stats)
tasks_params = (
(consts.TASK_NAMES.create_stats_user, True),
(consts.TASK_NAMES.remove_stats_user, False)
)
resp = self.app.patch(
reverse('MasterNodeSettingsHandler'),
headers=self.default_headers,
params='{}'
)
self.assertEqual(200, resp.status_code)
for task_name, must_send_stats in tasks_params:
task_count = objects.TaskCollection.filter_by(
None, name=consts.TASK_NAMES.create_stats_user).count()
self.assertEqual(0, task_count)
with mock.patch('nailgun.objects.MasterNodeSettings.'
'must_send_stats', return_value=must_send_stats):
for cluster_status in consts.CLUSTER_STATUSES:
if cluster_status == consts.CLUSTER_STATUSES.operational:
continue
cluster.status = cluster_status
self.env.db().flush()
resp = self.app.patch(
reverse('MasterNodeSettingsHandler'),
headers=self.default_headers,
params='{}'
)
self.assertEqual(200, resp.status_code)
task_count = objects.TaskCollection.filter_by(
None, name=task_name).count()
self.assertEqual(0, task_count)
def test_create_stats_user_not_required(self):
with mock.patch('nailgun.objects.MasterNodeSettings.must_send_stats',
@ -149,3 +167,58 @@ class TestStatsUserTaskManagers(BaseMasterNodeSettignsTest):
)
self.assertEqual(200, resp.status_code)
self.assertTrue(executer.called)
@fake_tasks(override_state={'progress': 100,
'status': consts.TASK_STATUSES.ready})
def test_remove_stats_user(self):
self.env.create(
nodes_kwargs=[
{'roles': ['controller'], 'pending_addition': True},
{'roles': ['controller'], 'pending_addition': True},
{'roles': ['controller'], 'pending_addition': True},
]
)
with mock.patch('nailgun.objects.MasterNodeSettings.must_send_stats',
return_value=True):
deploy_task = self.env.launch_deployment()
self.env.wait_ready(deploy_task)
with mock.patch('nailgun.objects.MasterNodeSettings.must_send_stats',
return_value=False):
resp = self.app.patch(
reverse('MasterNodeSettingsHandler'),
headers=self.default_headers,
params=jsonutils.dumps({})
)
self.assertEqual(200, resp.status_code)
task = objects.TaskCollection.filter_by(
None, name=consts.TASK_NAMES.remove_stats_user).first()
self.assertIsNotNone(task)
def test_remove_stats_user_not_required(self):
with mock.patch('nailgun.objects.MasterNodeSettings.must_send_stats',
return_value=True):
with mock.patch('nailgun.task.manager.RemoveStatsUserTaskManager.'
'execute') as executer:
resp = self.app.patch(
reverse('MasterNodeSettingsHandler'),
headers=self.default_headers,
params=jsonutils.dumps({})
)
self.assertEqual(200, resp.status_code)
self.assertFalse(executer.called)
def test_remove_stats_user_called(self):
with mock.patch('nailgun.objects.MasterNodeSettings.must_send_stats',
return_value=False):
with mock.patch('nailgun.task.manager.RemoveStatsUserTaskManager.'
'execute') as executer:
resp = self.app.patch(
reverse('MasterNodeSettingsHandler'),
headers=self.default_headers,
params=jsonutils.dumps({})
)
self.assertEqual(200, resp.status_code)
self.assertTrue(executer.called)