First commit of devstack directory
Add devstack files and directories below. - Document: README.rst - Entrypoint: plugin.sh - Functions Library: lib/slogging - Variable Settings: settings - Configuration: files/log-processor.conf - Cron Settings: files/cron.d/* Change-Id: I98df6519ede6916e41579b2318bdc71c2b15df94
This commit is contained in:
Hajime Kondo
parent
7444891218
commit
cf8ac8df05
|
|
@ -0,0 +1,28 @@
|
|||
===============================
|
||||
Enabling Slogging in DevStack
|
||||
===============================
|
||||
|
||||
1. Download Devstack::
|
||||
|
||||
git clone https://git.openstack.org/openstack-dev/devstack
|
||||
cd devstack
|
||||
|
||||
2. Add this repo as an external repository in ``local.conf`` file::
|
||||
|
||||
[[local|localrc]]
|
||||
enable_plugin slogging https://git.openstack.org/openstack/slogging
|
||||
|
||||
To use stable branches, make sure devstack is on that branch, and specify
|
||||
the branch name to enable_plugin, for example::
|
||||
|
||||
[[local|localrc]]
|
||||
enable_plugin slogging https://git.openstack.org/openstack/slogging <refspec>
|
||||
|
||||
In case triggered by cron, set environ variable ``SLOGGING_CRON``::
|
||||
|
||||
[[local|localrc]]
|
||||
SLOGGING_CRON=True
|
||||
|
||||
3. Run ``stack.sh``::
|
||||
|
||||
./stack.sh
|
||||
|
|
@ -0,0 +1 @@
|
|||
5 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-uploader %SLOGGING_BASE%/etc/log-processor.conf access
|
||||
|
|
@ -0,0 +1 @@
|
|||
5 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-container-stats-logger %SLOGGING_BASE%/etc/log-processor.conf
|
||||
|
|
@ -0,0 +1 @@
|
|||
0 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-account-stats-logger %SLOGGING_BASE%/etc/log-processor.conf
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
10 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-uploader %SLOGGING_BASE%/etc/log-processor.conf stats
|
||||
15 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-uploader %SLOGGING_BASE%/etc/log-processor.conf container-stats
|
||||
|
|
@ -0,0 +1 @@
|
|||
30 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-stats-collector %SLOGGING_BASE%/etc/log-processor.conf
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
[DEFAULT]
|
||||
swift_account = %TEMPACCOUNT%
|
||||
user = %STACK_USER%
|
||||
log_level = DEBUG
|
||||
log_facility = LOG_LOCAL1
|
||||
|
||||
[log-processor]
|
||||
container_name = log_processing_data
|
||||
proxy_server_conf = /etc/swift/proxy-server.conf
|
||||
lookback_hours = 120
|
||||
lockback_window = 120
|
||||
format_type = json
|
||||
|
||||
[log-processor-access]
|
||||
container_name = log_data
|
||||
log_dir = %DEST%/data/swift/logs/hourly/
|
||||
source_filename_pattern = ^
|
||||
(?P<year>[0-9]{4})
|
||||
(?P<month>[0-1][0-9])
|
||||
(?P<day>[0-3][0-9])
|
||||
(?P<hour>[0-2][0-9])
|
||||
.*$
|
||||
class_path = slogging.access_processor.AccessLogProcessor
|
||||
unlink_log = True
|
||||
new_log_cutoff = 30
|
||||
|
||||
[log-processor-stats]
|
||||
container_name = account_stats
|
||||
log_dir = %DEST%/data/swift/logs/stats/
|
||||
class_path = slogging.stats_processor.StatsLogProcessor
|
||||
log_name = stats-%Y%m%d%H_
|
||||
stats_type = account
|
||||
account_server_data_dir = accounts
|
||||
devices = %DEST%/data/swift/1
|
||||
mount_check = false
|
||||
unlink_log = True
|
||||
new_log_cutoff = 30
|
||||
|
||||
[log-processor-container-stats]
|
||||
container_name = container_stats
|
||||
log_dir = %DEST%/data/swift/logs/stats/
|
||||
class_path = slogging.stats_processor.StatsLogProcessor
|
||||
log_name = container-stats-%Y%m%d%H_
|
||||
stats_type = container
|
||||
container_server_data_dir = containers
|
||||
processable = false
|
||||
devices = %DEST%/data/swift/1
|
||||
mount_check = false
|
||||
unlink_log = True
|
||||
new_log_cutoff = 30
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# lib/slogging
|
||||
# Functions to control the configuration and operation of the slogging plugin
|
||||
|
||||
# Dependencies:
|
||||
#
|
||||
# - Swift service has already installed into $DEST/swift with below in local.conf
|
||||
# - ``s-proxy``, ``s-container``, ``s-object``, ``s-object``
|
||||
|
||||
# - ``functions`` file
|
||||
# - ``DEST``, ``$STACK_USER``must be defined
|
||||
|
||||
# - ``lib/swift`` file
|
||||
# - ``SWIFT_CONFIG_PROXY_SERVER`` must be defined
|
||||
|
||||
# ``stack.sh`` file
|
||||
# - ``echo_summary`` function must be defined
|
||||
#
|
||||
|
||||
# ``stack.sh`` calls the entry points in this order:
|
||||
#
|
||||
# - install_slogging
|
||||
# - configure_slogging
|
||||
# - verify_swift
|
||||
# - metering-sample_slogging
|
||||
|
||||
function _get_token_and_endpoint {
|
||||
curl -v -H "X-Storage-User:${SLOGGING['tempuser']}" \
|
||||
-H "X-Storage-Pass:${SLOGGING['temppass']}" \
|
||||
${SLOGGING['auth_url']} 2>&1 | \
|
||||
egrep 'X-Storage-Url:|X-Auth-Token:' | \
|
||||
sed -e 's|: |=|g' -e 's|<||g' -e 's|\-|_|g' -e 's|\r||g'
|
||||
}
|
||||
|
||||
function _exec_swift_api {
|
||||
for envvar in $(set | awk -F'=' '/^OS_/{print $1}'); do
|
||||
unset ${envvar}
|
||||
done
|
||||
swift -A ${SLOGGING['auth_url']} \
|
||||
-U ${SLOGGING['tempuser']} \
|
||||
-K ${SLOGGING['temppass']} stat
|
||||
}
|
||||
|
||||
function install_slogging {
|
||||
pushd $SLOGGING_BASE && {
|
||||
sudo python setup.py install
|
||||
} && popd
|
||||
}
|
||||
|
||||
function configure_slogging {
|
||||
local saio_base="$DEST/swift/doc/saio"
|
||||
local rsyslog_conf="rsyslog.d/10-swift.conf"
|
||||
local swift_log_base="$DEST/data/swift"
|
||||
|
||||
# Set config
|
||||
if [ ! -f "/etc/$rsyslog_conf" ]; then
|
||||
sudo cp $saio_base/$rsyslog_conf /etc/$rsyslog_conf
|
||||
fi
|
||||
sudo sed -e "
|
||||
s|/var/log/swift|$swift_log_base/logs|g;
|
||||
s|#\$templat|\$templat|g;
|
||||
s|#local1\.|local1.|g;
|
||||
" -i /etc/$rsyslog_conf
|
||||
sudo sed -e '/PrivDropToGroup/c $PrivDropToGroup adm' /etc/rsyslog.conf
|
||||
|
||||
# Set directory
|
||||
sudo mkdir -p $swift_log_base/logs/hourly
|
||||
safe_chown -R $STACK_USER.adm $swift_log_base
|
||||
safe_chmod 775 $swift_log_base/logs $swift_log_base/logs/hourly
|
||||
add_user_to_group syslog adm
|
||||
add_user_to_group $STACK_USER adm
|
||||
|
||||
# Reflect daemon
|
||||
restart_service rsyslog
|
||||
|
||||
# Generate /etc/swift/log-processor.conf
|
||||
local log_processor=$SLOGGING_BASE/etc/log-processor.conf
|
||||
sudo cp $SLOGGING_BASE/devstack/files/log-processor.conf $log_processor
|
||||
|
||||
# set device
|
||||
iniset $log_processor log-processor-container-stats devices $swift_log_base/1
|
||||
iniset $log_processor log-processor-stats devices $swift_log_base/1
|
||||
|
||||
# set log_dir
|
||||
iniset $log_processor log-processor-container-stats log_dir $swift_log_base/logs/stats/
|
||||
iniset $log_processor log-processor-stats log_dir $swift_log_base/logs/stats/
|
||||
iniset $log_processor log-processor-access log_dir $swift_log_base/logs/hourly/
|
||||
|
||||
iniset $log_processor DEFAULT swift_account ${SLOGGING['tempaccount']}
|
||||
iniset $log_processor DEFAULT user $STACK_USER
|
||||
iniset $SWIFT_CONFIG_PROXY_SERVER app:proxy-server log_facility LOG_LOCAL1
|
||||
|
||||
# Set cron
|
||||
if [ "$SLOGGING_CRON" == "True" ]; then
|
||||
sudo cp $SLOGGING_BASE/devstack/files/cron.d/swift-* /etc/cron.d/
|
||||
sudo sed -e "
|
||||
s|%STACK_USER%|$STACK_USER|g;
|
||||
s|%SLOGGING_BASE%|$SLOGGING_BASE|g;
|
||||
" -i /etc/cron.d/swift-*
|
||||
else
|
||||
sudo rm -f /etc/cron.d/swift-*
|
||||
fi
|
||||
}
|
||||
|
||||
function verify_swift {
|
||||
# Internal proxy(identity) API
|
||||
for var in $(_get_token_and_endpoint); do export $var; done
|
||||
curl -v -H "X-Auth-Token: $X_Auth_Token" $X_Storage_Url
|
||||
|
||||
# Swift API
|
||||
local retry=3
|
||||
local interval=5
|
||||
for i in {1..$retry}; do
|
||||
( _exec_swift_api ) && return 0
|
||||
sleep $interval
|
||||
done
|
||||
return -1
|
||||
}
|
||||
|
||||
function metering-sample_slogging {
|
||||
echo_summary "Swift's Function Test"
|
||||
$DEST/swift/.functests
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
# plugin.sh - Devstack plugin.sh script to install and configure slogging with swift settings
|
||||
|
||||
# Save trace setting
|
||||
XTRACE=$(set +o | grep xtrace)
|
||||
set -o xtrace
|
||||
|
||||
echo_summary "slogging's plugin.sh was called..."
|
||||
source $DEST/slogging/devstack/lib/slogging
|
||||
|
||||
# check for service enabled
|
||||
if is_plugin_enabled slogging; then
|
||||
|
||||
if [[ "$1" == "stack" && "$2" == "pre-install" ]]; then
|
||||
# Set up system services # apt-get install, pip insall
|
||||
#echo_summary "Configuring system services Slogging"
|
||||
#install_package cowsay
|
||||
:
|
||||
|
||||
elif [[ "$1" == "stack" && "$2" == "install" ]]; then
|
||||
# Perform installation of service source # python setup.py install
|
||||
#echo_summary "Installing Slogging"
|
||||
#install_slogging
|
||||
:
|
||||
|
||||
elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
|
||||
# Configure after the other layer 1 and 2 services have been configured
|
||||
echo_summary "Configuring Slogging" # memcached, rsyslog, ... etc
|
||||
configure_slogging
|
||||
|
||||
elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
|
||||
# Initialize and start the
|
||||
echo_summary "Verifying Swift nomally installed"
|
||||
verify_swift
|
||||
|
||||
elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
|
||||
# Unit & Function test for slogging to get swift log
|
||||
echo_summary "Metering Sample with Swift"
|
||||
metering-sample_slogging
|
||||
fi
|
||||
|
||||
if [[ "$1" == "unstack" ]]; then
|
||||
# Shut down template services
|
||||
# no-op
|
||||
:
|
||||
fi
|
||||
|
||||
if [[ "$1" == "clean" ]]; then
|
||||
# Remove state and transient data
|
||||
# Remember clean.sh first calls unstack.sh
|
||||
# no-op
|
||||
:
|
||||
fi
|
||||
fi
|
||||
|
||||
# Restore
|
||||
$XTRACE
|
||||
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
# settings file for sloggin plugin
|
||||
declare -A -g SLOGGING
|
||||
SLOGGING['tempuser']="test:tester"
|
||||
SLOGGING['temppass']="testing"
|
||||
SLOGGING['tempaccount']="TEMPAUTH_test"
|
||||
SLOGGING['auth_url']="http://127.0.0.1:8080/auth/v1.0"
|
||||
SLOGGING_BASE=$DEST/slogging
|
||||
|
||||
enable_service s-proxy s-object s-container s-account
|
||||
Loading…
Reference in New Issue