[WIP] Implement Slack notification plugin
Right now, Freezer-DR notification engine only supports email. This patch add support for sending notification to Slack channels. Change-Id: Ie19f71e9922750fcb295b7c808d423b2c01a1fc5
This commit is contained in:
parent
650bbc5755
commit
4ae1693bd0
|
@ -31,8 +31,8 @@ _MONITORS = [
|
|||
'StandardDriver',
|
||||
help='Driver used to get a status updates of compute nodes'),
|
||||
cfg.StrOpt('backend_name',
|
||||
help="configuration section name. This should contain your "
|
||||
"monitoring specific configuration options.")
|
||||
help='configuration section name. This should contain your '
|
||||
'monitoring specific configuration options.')
|
||||
]
|
||||
|
||||
_COMMON = [
|
||||
|
@ -44,97 +44,100 @@ _COMMON = [
|
|||
_FENCER = [
|
||||
cfg.StrOpt('credentials-file',
|
||||
help='YAML File contains the required credentials for compute '
|
||||
'nodes'),
|
||||
'nodes'),
|
||||
cfg.IntOpt('retries',
|
||||
default=1,
|
||||
help='Number of retries to fence the each compute node. Must be'
|
||||
' at least 1 to try first the soft shutdown'),
|
||||
' at least 1 to try first the soft shutdown'),
|
||||
cfg.IntOpt('hold-period',
|
||||
default=10,
|
||||
help='Time in seconds to wait between retries. Should be '
|
||||
'reasonable amount of time as different servers take '
|
||||
'different times to shut off'),
|
||||
'reasonable amount of time as different servers take '
|
||||
'different times to shut off'),
|
||||
cfg.StrOpt('driver',
|
||||
default='freezer_dr.fencers.drivers.ipmi.driver.IpmiDriver',
|
||||
help='Choose the best fencer driver i.e.(ipmi, libvirt, ..'),
|
||||
cfg.DictOpt('options',
|
||||
default={},
|
||||
help='List of kwargs to customize the fencer operation. You '
|
||||
'fencer driver should support these options. Options '
|
||||
'should be in key:value format')
|
||||
'fencer driver should support these options. Options '
|
||||
'should be in key:value format')
|
||||
]
|
||||
|
||||
_KEYSTONE_AUTH_TOKEN = [
|
||||
cfg.StrOpt('auth_uri',
|
||||
help='OpenStack auth URI i.e. http://controller:5000',
|
||||
dest='auth_uri'), cfg.StrOpt(
|
||||
'auth_url',
|
||||
help='OpenStack auth URL i.e. http://controller:35357/v3',
|
||||
dest='auth_url'),
|
||||
dest='auth_uri'),
|
||||
cfg.StrOpt('auth_url',
|
||||
help='OpenStack auth URL i.e. http://controller:35357/v3',
|
||||
dest='auth_url'),
|
||||
cfg.StrOpt('auth_plugin',
|
||||
help='OpenStack auth plugin i.e. ( password, token, ...) '
|
||||
'password is the only available plugin for the time being',
|
||||
dest='auth_plugin'), cfg.StrOpt('username',
|
||||
help='OpenStack username',
|
||||
dest='username'),
|
||||
'password is the only available plugin for the time '
|
||||
'being',
|
||||
dest='auth_plugin'),
|
||||
cfg.StrOpt('username',
|
||||
help='OpenStack username',
|
||||
dest='username'),
|
||||
cfg.StrOpt('password',
|
||||
help='OpenStack Password',
|
||||
dest='password'), cfg.StrOpt('project_name',
|
||||
help='OpenStack Project Name.',
|
||||
dest='project_name'),
|
||||
dest='password'),
|
||||
cfg.StrOpt('project_name',
|
||||
help='OpenStack Project Name.',
|
||||
dest='project_name'),
|
||||
cfg.StrOpt('domain_name',
|
||||
help='OpenStack domain Name.',
|
||||
dest='domain_name'), cfg.StrOpt(
|
||||
'project_domain_id',
|
||||
help='OpenStack Project Domain id, default is Default',
|
||||
dest='project_domain_id'),
|
||||
dest='domain_name'),
|
||||
cfg.StrOpt('project_domain_id',
|
||||
help='OpenStack Project Domain id, default is Default',
|
||||
dest='project_domain_id'),
|
||||
cfg.StrOpt('user_domain_id',
|
||||
help='OpenStack user Domain id, default is Default',
|
||||
dest='user_domain_id'), cfg.StrOpt(
|
||||
'project_domain_name',
|
||||
help='OpenStack Project Domain name, default is Default',
|
||||
dest='project_domain_name'), cfg.StrOpt(
|
||||
'user_domain_name',
|
||||
help='OpenStack user Domain name, default is Default',
|
||||
dest='user_domain_name'),
|
||||
dest='user_domain_id'),
|
||||
cfg.StrOpt('project_domain_name',
|
||||
help='OpenStack Project Domain name, default is Default',
|
||||
dest='project_domain_name'),
|
||||
cfg.StrOpt('user_domain_name',
|
||||
help='OpenStack user Domain name, default is Default',
|
||||
dest='user_domain_name'),
|
||||
cfg.DictOpt('kwargs',
|
||||
help='OpenStack Authentication arguments you can pass it here '
|
||||
'as Key:Value, Key1:Value1, ... ',
|
||||
help='OpenStack Authentication arguments you can pass it here'
|
||||
' as Key:Value, Key1:Value1, ... ',
|
||||
dest='kwargs',
|
||||
default={})
|
||||
]
|
||||
|
||||
_EVACUATION = [
|
||||
cfg.StrOpt(
|
||||
'driver',
|
||||
default='freezer_dr.evacuators.drivers.default.standard.'
|
||||
'StandardEvacuator',
|
||||
help='Time in seconds to wait between retries to disable compute'
|
||||
' node or put it in maintenance mode. Default 10 seconds',
|
||||
dest='driver'), cfg.IntOpt(
|
||||
'wait',
|
||||
default=10,
|
||||
help='Time in seconds to wait between retries to disable compute'
|
||||
' node or put it in maintenance mode. Default 10 seconds',
|
||||
dest='wait'),
|
||||
cfg.StrOpt('driver',
|
||||
default='freezer_dr.evacuators.drivers.default.standard.'
|
||||
'StandardEvacuator',
|
||||
help='Time in seconds to wait between retries to disable '
|
||||
'compute node or put it in maintenance mode. Default '
|
||||
'10 seconds',
|
||||
dest='driver'),
|
||||
cfg.IntOpt('wait',
|
||||
default=10,
|
||||
help='Time in seconds to wait between retries to disable '
|
||||
'compute node or put it in maintenance mode. Default '
|
||||
'10 seconds',
|
||||
dest='wait'),
|
||||
cfg.IntOpt('retries',
|
||||
default=1,
|
||||
help='Number of retries to put node in maintenance mode before '
|
||||
'reporting failure to evacuate the node',
|
||||
help='Number of retries to put node in maintenance mode before'
|
||||
' reporting failure to evacuate the node',
|
||||
dest='retries'),
|
||||
cfg.BoolOpt('shared-storage',
|
||||
default=False,
|
||||
help='Set this option to True in case your compute nodes are '
|
||||
'running on a shared storage or False if not',
|
||||
dest='shared_storage'),
|
||||
cfg.DictOpt(
|
||||
'options',
|
||||
default={},
|
||||
help='Dict contains kwargs to be passed to the evacuator driver'
|
||||
'. In case you have additional args needs to be passed to '
|
||||
'your evacuator please, list them as key0:value0, '
|
||||
'key1:value1, ....',
|
||||
dest='options')
|
||||
cfg.DictOpt('options',
|
||||
default={},
|
||||
help='Dict contains kwargs to be passed to the evacuator '
|
||||
'driver. In case you have additional args needs to be '
|
||||
'passed to your evacuator please, list them as '
|
||||
'key0:value0, key1:value1, ...',
|
||||
dest='options')
|
||||
]
|
||||
|
||||
_NOTIFIERS = [
|
||||
|
@ -143,48 +146,52 @@ _NOTIFIERS = [
|
|||
'StandardEmail',
|
||||
dest='driver',
|
||||
help='Notification driver to load it to notify users '
|
||||
'if something went wrong'),
|
||||
'if something went wrong. There are two supported drivers'
|
||||
': freezer_dr.notifiers.drivers.default.default_email.'
|
||||
'StandardEmail and freezer_dr.notifiers.drivers.default.'
|
||||
'slack.slack.SlackNotifier'),
|
||||
cfg.StrOpt('endpoint',
|
||||
default=None,
|
||||
dest='endpoint',
|
||||
help='Endpoint URL for the notification system. If you the '
|
||||
'driver you are using doesnot require any URL just comment '
|
||||
'it or use none'),
|
||||
cfg.StrOpt(
|
||||
'username',
|
||||
default=None,
|
||||
dest='username',
|
||||
help='Username to authenticate against the notification system. '
|
||||
'If the driver you are using doesnot require any '
|
||||
'authentications comment or use None'), cfg.StrOpt(
|
||||
'password',
|
||||
default=None,
|
||||
dest='password',
|
||||
help='Password to authenticate against the notification system. '
|
||||
'If the driver you are using doesnot require any '
|
||||
'authentications comment or use None'),
|
||||
'driver you are using doesnot require any URL just comment'
|
||||
' it or use none'),
|
||||
cfg.StrOpt('username',
|
||||
default=None,
|
||||
dest='username',
|
||||
help='Username to authenticate against the notification system.'
|
||||
' If the driver you are using doesnot require any '
|
||||
'authentications comment or use None'),
|
||||
cfg.StrOpt('password',
|
||||
default=None,
|
||||
dest='password',
|
||||
help='Password to authenticate against the notification system. '
|
||||
'If the driver you are using doesnot require any '
|
||||
'authentications comment or use None'),
|
||||
cfg.StrOpt('templates-dir',
|
||||
dest='templates-dir',
|
||||
default='/etc/freezer/templates',
|
||||
help='Path to Jinja2 templates directory that contains '
|
||||
'message templates'),
|
||||
'message templates'),
|
||||
cfg.DictOpt('options',
|
||||
default={},
|
||||
dest='options',
|
||||
help='Key:Value Kwargs to pass it to the notification driver, '
|
||||
'if you want to pass any special arguments for your '
|
||||
'driver. '),
|
||||
default={},
|
||||
dest='options',
|
||||
help='Key:Value Kwargs to pass it to the notification driver, '
|
||||
'if you want to pass any special arguments for your '
|
||||
'driver. If you want to use the SlackNotifier driver, '
|
||||
'set as: options = slack_timeout:512,'
|
||||
'slack_ca_certs:/ca.crt,slack_insecured:True'),
|
||||
cfg.ListOpt('notify-list',
|
||||
default=[],
|
||||
dest='notify-list',
|
||||
help='List of emails to sent them notification if something '
|
||||
'went wrong and Freezer DR wasnot able to send an email to the '
|
||||
'tenant admin'),
|
||||
default=[],
|
||||
dest='notify-list',
|
||||
help='List of emails to sent them notification if something '
|
||||
'went wrong and Freezer DR wasnot able to send an email '
|
||||
'to the tenant admin'),
|
||||
cfg.StrOpt('notify-from',
|
||||
dest='notify-from',
|
||||
help='The sender address, it can be email address if we used '
|
||||
'default email driver, or phone number if we use sms '
|
||||
'gateway for example.')
|
||||
'default email driver, or phone number if we use sms '
|
||||
'gateway for example.')
|
||||
]
|
||||
|
||||
|
||||
|
@ -194,73 +201,73 @@ def build_os_options():
|
|||
cfg.StrOpt('os-username',
|
||||
default=env('OS_USERNAME'),
|
||||
help='Name used for authentication with the OpenStack '
|
||||
'Identity service. Defaults to env[OS_USERNAME].',
|
||||
'Identity service. Defaults to env[OS_USERNAME].',
|
||||
dest='os_username'),
|
||||
cfg.StrOpt('os-password',
|
||||
default=env('OS_PASSWORD'),
|
||||
help='Password used for authentication with the OpenStack '
|
||||
'Identity service. Defaults to env[OS_PASSWORD].',
|
||||
'Identity service. Defaults to env[OS_PASSWORD].',
|
||||
dest='os_password'),
|
||||
cfg.StrOpt('os-project-name',
|
||||
default=env('OS_PROJECT_NAME'),
|
||||
help='Project name to scope to. Defaults to '
|
||||
'env[OS_PROJECT_NAME].',
|
||||
'env[OS_PROJECT_NAME].',
|
||||
dest='os_project_name'),
|
||||
cfg.StrOpt('os-project-domain-name',
|
||||
default=env('OS_PROJECT_DOMAIN_NAME'),
|
||||
help='Domain name containing project. Defaults to '
|
||||
'env[OS_PROJECT_DOMAIN_NAME].',
|
||||
'env[OS_PROJECT_DOMAIN_NAME].',
|
||||
dest='os_project_domain_name'),
|
||||
cfg.StrOpt('os-user-domain-name',
|
||||
default=env('OS_USER_DOMAIN_NAME'),
|
||||
help='User\'s domain name. Defaults to '
|
||||
'env[OS_USER_DOMAIN_NAME].',
|
||||
'env[OS_USER_DOMAIN_NAME].',
|
||||
dest='os_user_domain_name'),
|
||||
cfg.StrOpt('os-tenant-name',
|
||||
default=env('OS_TENANT_NAME'),
|
||||
help='Tenant to request authorization on. Defaults to '
|
||||
'env[OS_TENANT_NAME].',
|
||||
'env[OS_TENANT_NAME].',
|
||||
dest='os_tenant_name'),
|
||||
cfg.StrOpt('os-tenant-id',
|
||||
default=env('OS_TENANT_ID'),
|
||||
help='Tenant to request authorization on. Defaults to '
|
||||
'env[OS_TENANT_ID].',
|
||||
'env[OS_TENANT_ID].',
|
||||
dest='os_tenant_id'),
|
||||
cfg.StrOpt('os-auth-url',
|
||||
default=env('OS_AUTH_URL'),
|
||||
help='Specify the Identity endpoint to use for '
|
||||
'authentication. Defaults to env[OS_AUTH_URL].',
|
||||
'authentication. Defaults to env[OS_AUTH_URL].',
|
||||
dest='os_auth_url'),
|
||||
cfg.StrOpt('os-backup-url',
|
||||
default=env('OS_BACKUP_URL'),
|
||||
help='Specify the Freezer backup service endpoint to use. '
|
||||
'Defaults to env[OS_BACKUP_URL].',
|
||||
'Defaults to env[OS_BACKUP_URL].',
|
||||
dest='os_backup_url'),
|
||||
cfg.StrOpt('os-region-name',
|
||||
default=env('OS_REGION_NAME'),
|
||||
help='Specify the region to use. Defaults to '
|
||||
'env[OS_REGION_NAME].',
|
||||
'env[OS_REGION_NAME].',
|
||||
dest='os_region_name'),
|
||||
cfg.StrOpt(
|
||||
'os-token',
|
||||
default=env('OS_TOKEN'),
|
||||
help='Specify an existing token to use instead of retrieving'
|
||||
' one via authentication (e.g. with username & '
|
||||
'password). Defaults to env[OS_TOKEN].',
|
||||
dest='os_token'),
|
||||
cfg.StrOpt('os-token',
|
||||
default=env('OS_TOKEN'),
|
||||
help='Specify an existing token to use instead of '
|
||||
'retrieving one via authentication (e.g. '
|
||||
'with username & password). Defaults to '
|
||||
'env[OS_TOKEN].',
|
||||
dest='os_token'),
|
||||
cfg.StrOpt('os-identity-api-version',
|
||||
default=env('OS_IDENTITY_API_VERSION'),
|
||||
help='Identity API version: 2.0 or 3. '
|
||||
'Defaults to env[OS_IDENTITY_API_VERSION]',
|
||||
'Defaults to env[OS_IDENTITY_API_VERSION]',
|
||||
dest='os_identity_api_version'),
|
||||
cfg.StrOpt('os-endpoint-type',
|
||||
choices=['public', 'publicURL', 'internal', 'internalURL',
|
||||
'admin', 'adminURL'],
|
||||
default=env('OS_ENDPOINT_TYPE') or 'public',
|
||||
help='Endpoint type to select. Valid endpoint types: '
|
||||
'"public" or "publicURL", "internal" or "internalURL",'
|
||||
' "admin" or "adminURL". Defaults to '
|
||||
'env[OS_ENDPOINT_TYPE] or "public"',
|
||||
'"public" or "publicURL", "internal" or "internalURL"'
|
||||
', "admin" or "adminURL". Defaults to '
|
||||
'env[OS_ENDPOINT_TYPE] or "public"',
|
||||
dest='os_endpoint_type'),
|
||||
]
|
||||
|
||||
|
@ -274,14 +281,14 @@ def configure():
|
|||
monitors_grp = cfg.OptGroup('monitoring',
|
||||
title='Monitoring',
|
||||
help='Monitoring Driver/plugin to be used to '
|
||||
'monitor compute nodes')
|
||||
'monitor compute nodes')
|
||||
CONF.register_group(monitors_grp)
|
||||
CONF.register_opts(_MONITORS, group='monitoring')
|
||||
|
||||
fencers_grp = cfg.OptGroup('fencer',
|
||||
title='fencer Options',
|
||||
help='fencer Driver/plugin to be used to '
|
||||
'fence compute nodes')
|
||||
'fence compute nodes')
|
||||
CONF.register_group(fencers_grp)
|
||||
CONF.register_opts(_FENCER, group='fencer')
|
||||
|
||||
|
@ -289,7 +296,7 @@ def configure():
|
|||
evacuators_grp = cfg.OptGroup('evacuation',
|
||||
title='Evacuation Options',
|
||||
help='Evacuation Driver/plugin opts to be '
|
||||
'used to Evacuate compute nodes')
|
||||
'used to Evacuate compute nodes')
|
||||
CONF.register_group(evacuators_grp)
|
||||
CONF.register_opts(_EVACUATION, group='evacuation')
|
||||
|
||||
|
@ -297,8 +304,8 @@ def configure():
|
|||
notifiers_grp = cfg.OptGroup('notifiers',
|
||||
title='Notification Options',
|
||||
help='Notification Driver/plugin opts to be '
|
||||
'used to Notify admins/users if failure '
|
||||
'happens')
|
||||
'used to Notify admins/users if failure'
|
||||
' happens')
|
||||
CONF.register_group(notifiers_grp)
|
||||
CONF.register_opts(_NOTIFIERS, group='notifiers')
|
||||
|
||||
|
@ -306,7 +313,7 @@ def configure():
|
|||
keystone_grp = cfg.OptGroup('keystone_authtoken',
|
||||
title='Keystone Auth Options',
|
||||
help='OpenStack Credentials to call the nova '
|
||||
'APIs to evacuate ')
|
||||
'APIs to evacuate ')
|
||||
CONF.register_group(keystone_grp)
|
||||
CONF.register_opts(_KEYSTONE_AUTH_TOKEN, group='keystone_authtoken')
|
||||
|
||||
|
|
|
@ -0,0 +1,149 @@
|
|||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from oslo_log import log
|
||||
from freezer_dr.notifiers.common.driver import NotifierBaseDriver
|
||||
from datetime import date
|
||||
from six.moves import urllib
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class SlackNotifier(NotifierBaseDriver):
|
||||
|
||||
MAX_CACHE_SIZE = 100
|
||||
RESPONSE_OK = 'ok'
|
||||
|
||||
_raw_data_url_caches = []
|
||||
|
||||
def __init__(self, url, username, password, templates_dir, notify_from,
|
||||
admin_list=None, **kwargs):
|
||||
super(SlackNotifier, self).__init__(url, username, password,
|
||||
templates_dir, notify_from,
|
||||
admin_list, **kwargs)
|
||||
LOG.info('Initializing SlackNotifier driver @ {0}'.format(url))
|
||||
|
||||
self.slack_timeout = kwargs.get('slack_timeout', '')
|
||||
self.slack_ca_certs = kwargs.get('slack_ca_certs', '')
|
||||
self.slack_insecured = kwargs.get('slack_insecured', 'True')
|
||||
self.slack_proxy = kwargs.get('slack_proxy', '')
|
||||
|
||||
def _build_slack_message(self, node, status):
|
||||
"""Builds slack message body
|
||||
"""
|
||||
body = {
|
||||
'title': 'Host Evacuation status',
|
||||
'host': node.get('host'),
|
||||
'tenants': node.get('tenants'),
|
||||
'instances': node.get('instances'),
|
||||
'hypervisor': node.get('details'),
|
||||
'evacuation_time': date.fromtimestamp(time.time()),
|
||||
'status': status
|
||||
}
|
||||
slack_request = {}
|
||||
slack_request['text'] = json.dumps(body, indent=3)
|
||||
|
||||
return slack_request
|
||||
|
||||
def _check_response(self, result):
|
||||
if 'application/json' in result.headers.get('Content-Type'):
|
||||
response = result.json()
|
||||
if response.get(self.RESPONSE_OK):
|
||||
return True
|
||||
else:
|
||||
LOG.error('Received an error message when trying to send to slack. error={}'
|
||||
.format(response.get('error')))
|
||||
return False
|
||||
elif self.RESPONSE_OK == result.text:
|
||||
return True
|
||||
else:
|
||||
LOG.error('Received an error message when trying to send to slack. error={}'
|
||||
.format(result.text))
|
||||
return False
|
||||
|
||||
def _send_message(self, request_options):
|
||||
try:
|
||||
url = request_options.get('url')
|
||||
result = requests.post(**request_options)
|
||||
if result.status_code not in range(200, 300):
|
||||
LOG.error('Received an HTTP code {} when trying to post on URL {}.'
|
||||
.format(result.status_code, url))
|
||||
return False
|
||||
|
||||
# Slack returns 200 ok even if the token is invalid. Response has valid error message
|
||||
if self._check_response(result):
|
||||
LOG.info('Notification successfully posted.')
|
||||
return True
|
||||
|
||||
LOG.error('Failed to send to slack on URL {}.'.format(url))
|
||||
return False
|
||||
except Exception as err:
|
||||
LOG.error('Error trying to send to slack on URL {}. Detail: {}'
|
||||
.format(url, err))
|
||||
return False
|
||||
|
||||
|
||||
|
||||
def notify_status(self, node, status):
|
||||
"""Notify the Host Evacuation status via slack
|
||||
Posts on the given url
|
||||
"""
|
||||
|
||||
slack_message = self._build_slack_message(node, status)
|
||||
address = self.url
|
||||
address = address.replace('#', '%23')
|
||||
|
||||
parsed_url = urllib.parse.urlsplit(address)
|
||||
query_params = urllib.parse.parse_qs(parsed_url.query)
|
||||
url = urllib.parse.urljoin(address, urllib.parse.urlparse(address).path)
|
||||
|
||||
verify = self.slack_ca_certs or not self.slack_insecured
|
||||
|
||||
proxy = self.slack_proxy
|
||||
proxy_dict = None
|
||||
if proxy is not None:
|
||||
proxy_dict = {'https': proxy}
|
||||
|
||||
data_format_list = ['json', 'data']
|
||||
if url in SlackNotifier._raw_data_url_caches:
|
||||
data_format_list = ['data']
|
||||
|
||||
for data_format in data_format_list:
|
||||
LOG.info('Trying to send message to {} as {}'
|
||||
.format(url, data_format))
|
||||
request_options = {
|
||||
'url': url,
|
||||
'verify': verify,
|
||||
'params': query_params,
|
||||
'proxies': proxy_dict,
|
||||
'timeout': self.slack_timeout,
|
||||
data_format: slack_message
|
||||
}
|
||||
if self._send_message(request_options):
|
||||
if (data_format == 'data' and
|
||||
url not in SlackNotifier._raw_data_url_caches and
|
||||
len(SlackNotifier._raw_data_url_caches) < self.MAX_CACHE_SIZE):
|
||||
SlackNotifier._raw_data_url_caches.append(url)
|
||||
return True
|
||||
|
||||
LOG.info('Failed to send message to {} as {}'
|
||||
.format(url, data_format))
|
||||
return False
|
||||
|
||||
def notify(self, message):
|
||||
pass
|
||||
|
Loading…
Reference in New Issue