436 lines
14 KiB
Python
436 lines
14 KiB
Python
# Copyright 2013 Rackspace, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import copy
|
|
import errno
|
|
import glob
|
|
import io
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
|
|
from ironic_lib import utils as ironic_utils
|
|
from oslo_concurrency import processutils
|
|
from oslo_log import log as logging
|
|
from oslo_serialization import base64
|
|
from oslo_utils import units
|
|
from six.moves.urllib import parse
|
|
|
|
from ironic_python_agent import errors
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
# Agent parameters can be passed by kernel command-line arguments and/or
|
|
# by virtual media. Virtual media parameters passed would be available
|
|
# when the agent is started, but might not be available for re-reading
|
|
# later on because:
|
|
# * Virtual media might be exposed from Swift and swift temp url might
|
|
# expire.
|
|
# * Ironic might have removed the floppy image from Swift after starting
|
|
# the deploy.
|
|
#
|
|
# Even if it's available, there is no need to re-read from the device and
|
|
# /proc/cmdline again, because it is never going to change. So we cache the
|
|
# agent parameters that was passed (by proc/cmdline and/or virtual media)
|
|
# when we read it for the first time, and then use this cache.
|
|
AGENT_PARAMS_CACHED = dict()
|
|
|
|
|
|
COLLECT_LOGS_COMMANDS = {
|
|
'ps': ['ps', 'au'],
|
|
'df': ['df', '-a'],
|
|
'iptables': ['iptables', '-L'],
|
|
'ip_addr': ['ip', 'addr'],
|
|
}
|
|
|
|
|
|
def execute(*cmd, **kwargs):
|
|
"""Convenience wrapper around ironic_lib's execute() method.
|
|
|
|
Executes and logs results from a system command.
|
|
"""
|
|
return ironic_utils.execute(*cmd, **kwargs)
|
|
|
|
|
|
def try_execute(*cmd, **kwargs):
|
|
"""The same as execute but returns None on error.
|
|
|
|
Executes and logs results from a system command. See docs for
|
|
oslo_concurrency.processutils.execute for usage.
|
|
|
|
Instead of raising an exception on failure, this method simply
|
|
returns None in case of failure.
|
|
|
|
:param *cmd: positional arguments to pass to processutils.execute()
|
|
:param log_stdout: keyword-only argument: whether to log the output
|
|
:param **kwargs: keyword arguments to pass to processutils.execute()
|
|
:raises: UnknownArgumentError on receiving unknown arguments
|
|
:returns: tuple of (stdout, stderr) or None in some error cases
|
|
"""
|
|
try:
|
|
return execute(*cmd, **kwargs)
|
|
except (processutils.ProcessExecutionError, OSError) as e:
|
|
LOG.debug('Command failed: %s', e)
|
|
|
|
|
|
def _read_params_from_file(filepath):
|
|
"""Extract key=value pairs from a file.
|
|
|
|
:param filepath: path to a file containing key=value pairs separated by
|
|
whitespace or newlines.
|
|
:returns: a dictionary representing the content of the file
|
|
"""
|
|
with open(filepath) as f:
|
|
cmdline = f.read()
|
|
|
|
options = cmdline.split()
|
|
params = {}
|
|
for option in options:
|
|
if '=' not in option:
|
|
continue
|
|
k, v = option.split('=', 1)
|
|
params[k] = v
|
|
|
|
return params
|
|
|
|
|
|
def _get_vmedia_device():
|
|
"""Finds the device filename of the virtual media device using sysfs.
|
|
|
|
:returns: a string containing the filename of the virtual media device
|
|
"""
|
|
sysfs_device_models = glob.glob("/sys/class/block/*/device/model")
|
|
vmedia_device_model = "virtual media"
|
|
for model_file in sysfs_device_models:
|
|
try:
|
|
with open(model_file) as model_file_fobj:
|
|
if vmedia_device_model in model_file_fobj.read().lower():
|
|
vmedia_device = model_file.split('/')[4]
|
|
return vmedia_device
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _get_vmedia_params():
|
|
"""This method returns the parameters passed through virtual media floppy.
|
|
|
|
:returns: a partial dict of potential agent configuration parameters
|
|
:raises: VirtualMediaBootError when it cannot find the virtual media device
|
|
"""
|
|
parameters_file = "parameters.txt"
|
|
|
|
vmedia_device_file_lower_case = "/dev/disk/by-label/ir-vfd-dev"
|
|
vmedia_device_file_upper_case = "/dev/disk/by-label/IR-VFD-DEV"
|
|
if os.path.exists(vmedia_device_file_lower_case):
|
|
vmedia_device_file = vmedia_device_file_lower_case
|
|
elif os.path.exists(vmedia_device_file_upper_case):
|
|
vmedia_device_file = vmedia_device_file_upper_case
|
|
else:
|
|
|
|
# TODO(rameshg87): This block of code is there only for compatibility
|
|
# reasons (so that newer agent can work with older Ironic). Remove
|
|
# this after Liberty release.
|
|
vmedia_device = _get_vmedia_device()
|
|
if not vmedia_device:
|
|
msg = "Unable to find virtual media device"
|
|
raise errors.VirtualMediaBootError(msg)
|
|
|
|
vmedia_device_file = os.path.join("/dev", vmedia_device)
|
|
|
|
vmedia_mount_point = tempfile.mkdtemp()
|
|
try:
|
|
try:
|
|
stdout, stderr = execute("mount", vmedia_device_file,
|
|
vmedia_mount_point)
|
|
except processutils.ProcessExecutionError as e:
|
|
msg = ("Unable to mount virtual media device %(device)s: "
|
|
"%(error)s" % {'device': vmedia_device_file, 'error': e})
|
|
raise errors.VirtualMediaBootError(msg)
|
|
|
|
parameters_file_path = os.path.join(vmedia_mount_point,
|
|
parameters_file)
|
|
params = _read_params_from_file(parameters_file_path)
|
|
|
|
try:
|
|
stdout, stderr = execute("umount", vmedia_mount_point)
|
|
except processutils.ProcessExecutionError as e:
|
|
pass
|
|
finally:
|
|
try:
|
|
shutil.rmtree(vmedia_mount_point)
|
|
except Exception as e:
|
|
pass
|
|
|
|
return params
|
|
|
|
|
|
def _get_cached_params():
|
|
"""Helper method to get cached params to ease unit testing."""
|
|
return AGENT_PARAMS_CACHED
|
|
|
|
|
|
def _set_cached_params(params):
|
|
"""Helper method to set cached params to ease unit testing."""
|
|
global AGENT_PARAMS_CACHED
|
|
AGENT_PARAMS_CACHED = params
|
|
|
|
|
|
def get_agent_params():
|
|
"""Gets parameters passed to the agent via kernel cmdline or vmedia.
|
|
|
|
Parameters can be passed using either the kernel commandline or through
|
|
virtual media. If boot_method is vmedia, merge params provided via vmedia
|
|
with those read from the kernel command line.
|
|
|
|
Although it should never happen, if a variable is both set by vmedia and
|
|
kernel command line, the setting in vmedia will take precedence.
|
|
|
|
:returns: a dict of potential configuration parameters for the agent
|
|
"""
|
|
|
|
# Check if we have the parameters cached
|
|
params = _get_cached_params()
|
|
if not params:
|
|
params = _read_params_from_file('/proc/cmdline')
|
|
|
|
# If the node booted over virtual media, the parameters are passed
|
|
# in a text file within the virtual media floppy.
|
|
if params.get('boot_method') == 'vmedia':
|
|
vmedia_params = _get_vmedia_params()
|
|
params.update(vmedia_params)
|
|
|
|
# Cache the parameters so that it can be used later on.
|
|
_set_cached_params(params)
|
|
|
|
# Check to see if any deprecated parameters have been used
|
|
deprecated_params = {'lldp-timeout': 'ipa-lldp-timeout'}
|
|
for old_param, new_param in deprecated_params.items():
|
|
if params.get(old_param) is not None:
|
|
LOG.warning("The parameter '%s' has been deprecated. Please "
|
|
"use %s instead.", old_param, new_param)
|
|
|
|
return copy.deepcopy(params)
|
|
|
|
|
|
def normalize(string):
|
|
"""Return a normalized string.
|
|
|
|
Take a urlencoded value from Ironic and urldecode it.
|
|
|
|
:param string: a urlencoded string
|
|
:returns: a normalized version of passed in string
|
|
"""
|
|
return parse.unquote(string).lower().strip()
|
|
|
|
|
|
class AccumulatedFailures(object):
|
|
"""Object to accumulate failures without raising exception."""
|
|
|
|
def __init__(self, exc_class=RuntimeError):
|
|
self._failures = []
|
|
self._exc_class = exc_class
|
|
|
|
def add(self, fail, *fmt):
|
|
"""Add failure with optional formatting.
|
|
|
|
:param fail: exception or error string
|
|
:param fmt: formatting arguments (only if fail is a string)
|
|
"""
|
|
if fmt:
|
|
fail = fail % fmt
|
|
LOG.error('%s', fail)
|
|
self._failures.append(fail)
|
|
|
|
def get_error(self):
|
|
"""Get error string or None."""
|
|
if not self._failures:
|
|
return
|
|
|
|
msg = ('The following errors were encountered:\n%s'
|
|
% '\n'.join('* %s' % item for item in self._failures))
|
|
return msg
|
|
|
|
def raise_if_needed(self):
|
|
"""Raise exception if error list is not empty.
|
|
|
|
:raises: RuntimeError
|
|
"""
|
|
if self._failures:
|
|
raise self._exc_class(self.get_error())
|
|
|
|
def __nonzero__(self):
|
|
return bool(self._failures)
|
|
|
|
__bool__ = __nonzero__
|
|
|
|
def __repr__(self): # pragma: no cover
|
|
# This is for tests
|
|
if self:
|
|
return '<%s: %s>' % (self.__class__.__name__,
|
|
', '.join(self._failures))
|
|
else:
|
|
return '<%s: success>' % self.__class__.__name__
|
|
|
|
|
|
def guess_root_disk(block_devices, min_size_required=4 * units.Gi):
|
|
"""Find suitable disk provided that root device hints are not given.
|
|
|
|
If no hints are passed find the first device larger than min_size_required,
|
|
assume it is the OS disk
|
|
"""
|
|
# TODO(russellhaering): This isn't a valid assumption in
|
|
# all cases, is there a more reasonable default behavior?
|
|
block_devices.sort(key=lambda device: device.size)
|
|
if not block_devices or block_devices[-1].size < min_size_required:
|
|
raise errors.DeviceNotFound(
|
|
"No suitable device was found "
|
|
"for deployment - root device hints were not provided "
|
|
"and all found block devices are smaller than %iB."
|
|
% min_size_required)
|
|
for device in block_devices:
|
|
if device.size >= min_size_required:
|
|
return device
|
|
|
|
|
|
def is_journalctl_present():
|
|
"""Check if the journalctl command is present.
|
|
|
|
:returns: True if journalctl is present, False if not.
|
|
"""
|
|
try:
|
|
devnull = open(os.devnull)
|
|
subprocess.check_call(['journalctl', '--version'], stdout=devnull,
|
|
stderr=devnull)
|
|
except OSError as e:
|
|
if e.errno == errno.ENOENT:
|
|
return False
|
|
return True
|
|
|
|
|
|
def get_command_output(command):
|
|
"""Return the output of a given command.
|
|
|
|
:param command: The command to be executed.
|
|
:raises: CommandExecutionError if the execution of the command fails.
|
|
:returns: A BytesIO string with the output.
|
|
"""
|
|
try:
|
|
out, _ = execute(*command, binary=True, log_stdout=False)
|
|
except (processutils.ProcessExecutionError, OSError) as e:
|
|
error_msg = ('Failed to get the output of the command "%(command)s". '
|
|
'Error: %(error)s' % {'command': command, 'error': e})
|
|
LOG.error(error_msg)
|
|
raise errors.CommandExecutionError(error_msg)
|
|
return io.BytesIO(out)
|
|
|
|
|
|
def get_journalctl_output(lines=None, units=None):
|
|
"""Query the contents of the systemd journal.
|
|
|
|
:param lines: Maximum number of lines to retrieve from the
|
|
logs. If None, return everything.
|
|
:param units: A list with the names of the units we should
|
|
retrieve the logs from. If None retrieve the logs
|
|
for everything.
|
|
:returns: A log string.
|
|
"""
|
|
cmd = ['journalctl', '--full', '--no-pager', '-b']
|
|
if lines is not None:
|
|
cmd.extend(['-n', str(lines)])
|
|
if units is not None:
|
|
[cmd.extend(['-u', u]) for u in units]
|
|
|
|
return get_command_output(cmd)
|
|
|
|
|
|
def gzip_and_b64encode(io_dict=None, file_list=None):
|
|
"""Gzip and base64 encode files and BytesIO buffers.
|
|
|
|
:param io_dict: A dictionary containing whose the keys are the file
|
|
names and the value a BytesIO object.
|
|
:param file_list: A list of file path.
|
|
:returns: A gzipped and base64 encoded string.
|
|
"""
|
|
io_dict = io_dict or {}
|
|
file_list = file_list or []
|
|
|
|
with io.BytesIO() as fp:
|
|
with tarfile.open(fileobj=fp, mode='w:gz') as tar:
|
|
for fname in io_dict:
|
|
ioobj = io_dict[fname]
|
|
tarinfo = tarfile.TarInfo(name=fname)
|
|
tarinfo.size = ioobj.seek(0, 2)
|
|
tarinfo.mtime = time.time()
|
|
ioobj.seek(0)
|
|
tar.addfile(tarinfo, ioobj)
|
|
|
|
for f in file_list:
|
|
tar.add(f)
|
|
|
|
fp.seek(0)
|
|
return base64.encode_as_bytes(fp.getvalue())
|
|
|
|
|
|
def collect_system_logs(journald_max_lines=None):
|
|
"""Collect system logs.
|
|
|
|
Collect system logs, for distributions using systemd the logs will
|
|
come from journald. On other distributions the logs will come from
|
|
the /var/log directory and dmesg output.
|
|
|
|
:param journald_max_lines: Maximum number of lines to retrieve from
|
|
the journald. if None, return everything.
|
|
:returns: A tar, gzip base64 encoded string with the logs.
|
|
"""
|
|
|
|
def try_get_command_output(io_dict, file_name, command):
|
|
try:
|
|
io_dict[file_name] = get_command_output(command)
|
|
except errors.CommandExecutionError:
|
|
pass
|
|
|
|
io_dict = {}
|
|
file_list = []
|
|
if is_journalctl_present():
|
|
io_dict['journal'] = get_journalctl_output(lines=journald_max_lines)
|
|
else:
|
|
try_get_command_output(io_dict, 'dmesg', ['dmesg'])
|
|
file_list.append('/var/log')
|
|
|
|
for name, cmd in COLLECT_LOGS_COMMANDS.items():
|
|
try_get_command_output(io_dict, name, cmd)
|
|
|
|
return gzip_and_b64encode(io_dict=io_dict, file_list=file_list)
|
|
|
|
|
|
def get_ssl_client_options(conf):
|
|
"""Format SSL-related requests options.
|
|
|
|
:param conf: oslo_config CONF object
|
|
:returns: tuple of 'verify' and 'cert' values to pass to requests
|
|
"""
|
|
if conf.insecure:
|
|
verify = False
|
|
else:
|
|
verify = conf.cafile or True
|
|
if conf.certfile and conf.keyfile:
|
|
cert = (conf.certfile, conf.keyfile)
|
|
else:
|
|
cert = None
|
|
return verify, cert
|