From 8e3b52feac0926d0ecb05c1aaf540565bc870368 Mon Sep 17 00:00:00 2001 From: naichuans Date: Tue, 12 Jun 2018 10:10:11 +0000 Subject: [PATCH] os-xenapi: add utility to setup guest VM logs Add code to copy guest logs to /var/logs/xen/guest and set rotation Change-Id: Ied7dadad2c3b7b6e9a05cf49d1ad16eac5cf9204 --- os_xenapi/cmd/bootstrap.py | 2 + os_xenapi/tests/cmd/test_bootstrap.py | 6 +- os_xenapi/utils/consoles.py | 54 ++++++++++++++++++ os_xenapi/utils/rotate_xen_guest_logs.sh | 70 ++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 2 deletions(-) create mode 100755 os_xenapi/utils/consoles.py create mode 100755 os_xenapi/utils/rotate_xen_guest_logs.sh diff --git a/os_xenapi/cmd/bootstrap.py b/os_xenapi/cmd/bootstrap.py index db6f19f..9c299db 100644 --- a/os_xenapi/cmd/bootstrap.py +++ b/os_xenapi/cmd/bootstrap.py @@ -24,6 +24,7 @@ import sys from os_xenapi.utils.common_conf import enable_linux_bridge from os_xenapi.utils.common_function import setup_logging +from os_xenapi.utils.consoles import setup_guest_console_log from os_xenapi.utils.himn import config_himn from os_xenapi.utils.iptables import config_iptables from os_xenapi.utils.sshclient import SSHClient @@ -107,6 +108,7 @@ def main(): config_iptables(dom0_client) install_plugins_to_dom0(dom0_client) enable_linux_bridge(dom0_client) + setup_guest_console_log(dom0_client) # Gather XenAPI relative facts and save them into file. get_and_store_facts(dom0_client, facts_file) diff --git a/os_xenapi/tests/cmd/test_bootstrap.py b/os_xenapi/tests/cmd/test_bootstrap.py index bc57e4d..5132ea8 100644 --- a/os_xenapi/tests/cmd/test_bootstrap.py +++ b/os_xenapi/tests/cmd/test_bootstrap.py @@ -71,6 +71,7 @@ class GetXenapiFactsTestCase(base.TestCase): @mock.patch.object(bootstrap, '_parse_args') @mock.patch.object(bootstrap, 'SSHClient') + @mock.patch.object(bootstrap, 'setup_guest_console_log') @mock.patch.object(bootstrap, 'config_himn') @mock.patch.object(bootstrap, 'config_iptables') @mock.patch.object(bootstrap, 'install_plugins_to_dom0') @@ -78,8 +79,8 @@ class GetXenapiFactsTestCase(base.TestCase): @mock.patch.object(bootstrap, 'enable_linux_bridge') @mock.patch.object(bootstrap, 'setup_logging') def test_bootstrap(self, mock_setup_logging, mock_enable_lbr, mock_facts, - mock_plugin, mock_iptables, mock_himn, mock_client, - mock_parse): + mock_plugin, mock_iptables, mock_himn, mock_guest_log, + mock_client, mock_parse): fake_opts = {'himn-ip': '169.254.0.1', 'passwd': 'passwd', 'user-name': 'root'} @@ -96,3 +97,4 @@ class GetXenapiFactsTestCase(base.TestCase): bootstrap.DEF_XENAPI_FACTS_FILE) mock_enable_lbr.assert_called_with(mock.sentinel.sshclient) mock_setup_logging.assert_called_once_with(log_level=logging.DEBUG) + mock_guest_log.assert_called_once_with(mock.sentinel.sshclient) diff --git a/os_xenapi/utils/consoles.py b/os_xenapi/utils/consoles.py new file mode 100755 index 0000000..815223e --- /dev/null +++ b/os_xenapi/utils/consoles.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""console utils + +It contains the utilities relative to guest VM console logs collecting.""" +import os +import sys + +from os_xenapi.utils import sshclient +from os_xenapi.utils import xapi_plugin + +OS_XENAPI_PKG = 'os-xenapi' + + +def exit_with_error(err_msg): + sys.stderr.write(err_msg) + sys.exit(1) + + +def setup_guest_console_log(dom0_client): + "Install console logrotate script" + dom0_client.ssh('mkdir -p /var/log/xen/guest') + dom0_client.ssh('mkdir -p /opt/xensource/bin') + is_tmp_dir, os_xenapi_dir = xapi_plugin.get_os_xenapi_dir() + dom0_client.scp( + os_xenapi_dir + "/os_xenapi/utils/rotate_xen_guest_logs.sh", + os.path.join("/opt/xensource/bin/", 'rotate_xen_guest_logs.sh')) + dom0_client.ssh('''crontab - << CRONTAB +* * * * * /opt/xensource/bin/rotate_xen_guest_logs.sh >/dev/null 2>&1 +CRONTAB''') + + +if __name__ == '__main__': + if len(sys.argv) != 4: + exit_with_error("Wrong parameters input.") + dom0_himn_ip, user_name, password = sys.argv[1:] + try: + client = sshclient.SSHClient(dom0_himn_ip, user_name, password) + except Exception: + exit_with_error("Create connection failed, ip: %(dom0_himn_ip)s," + " user_name: %(user_name)s" % + {'dom0_himn_ip': dom0_himn_ip, 'user_name': user_name}) + setup_guest_console_log(client) diff --git a/os_xenapi/utils/rotate_xen_guest_logs.sh b/os_xenapi/utils/rotate_xen_guest_logs.sh new file mode 100755 index 0000000..0a31182 --- /dev/null +++ b/os_xenapi/utils/rotate_xen_guest_logs.sh @@ -0,0 +1,70 @@ +#!/bin/bash +set -eux + +# Script to rotate console logs +# +# Should be run on Dom0, with cron, every minute: +# * * * * * /root/rotate_xen_guest_logs.sh +# +# Should clear out the guest logs on every boot +# because the domain ids may get re-used for a +# different tenant after the reboot +# +# /var/log/xen/guest could be mounted into a +# small loopback device to stop any guest being +# able to fill dom0 file system + +log_dir="/var/log/xen/guest" +kb=1024 +mb=1048576 +max_size_bytes=$((1*$mb)) +truncated_size_bytes=$((5*$kb)) +syslog_tag='rotate_xen_guest_logs' + +log_file_base="${log_dir}/console." + +# Only delete log files older than this number of minutes +# to avoid a race where Xen creates the domain and starts +# logging before the XAPI VM start returns (and allows us +# to preserve the log file using last_dom_id) +min_logfile_age=10 + +# Ensure logging is setup correctly for all domains +xenstore-write /local/logconsole/@ "${log_file_base}%d" + +# Grab the list of logs now to prevent a race where the domain is +# started after we get the valid last_dom_ids, but before the logs are +# deleted. Add spaces to ensure we can do containment tests below +current_logs=$(find "$log_dir" -type f) + +# Ensure the last_dom_id is set + updated for all running VMs +for vm in $(xe vm-list power-state=running --minimal | tr ',' ' '); do + xe vm-param-set uuid=$vm other-config:last_dom_id=$(xe vm-param-get uuid=$vm param-name=dom-id) +done + +# Get the last_dom_id for all VMs +valid_last_dom_ids=$(xe vm-list params=other-config --minimal | tr ';,' '\n\n' | grep last_dom_id | sed -e 's/last_dom_id: //g' | xargs) +echo "Valid dom IDs: $valid_last_dom_ids" | /usr/bin/logger -t $syslog_tag + +# Remove old console files that do not correspond to valid last_dom_id's +allowed_consoles=".*console.\(${valid_last_dom_ids// /\\|}\)$" +delete_logs=`find "$log_dir" -type f -mmin +${min_logfile_age} -not -regex "$allowed_consoles"` +for log in $delete_logs; do + if echo "$current_logs" | grep -q -w "$log"; then + echo "Deleting: $log" | /usr/bin/logger -t $syslog_tag + rm $log + fi +done + +# Truncate all remaining logs +for log in `find "$log_dir" -type f -regex '.*console.*' -size +${max_size_bytes}c`; do + echo "Truncating log: $log" | /usr/bin/logger -t $syslog_tag + tmp="$log.tmp" + tail -c $truncated_size_bytes "$log" > "$tmp" + mv -f "$tmp" "$log" + + # Notify xen that it needs to reload the file + domid="${log##*.}" + xenstore-write /local/logconsole/$domid "$log" + xenstore-rm /local/logconsole/$domid +done