Merge "First commit of devstack directory"

This commit is contained in:
Zuul 2018-05-07 22:58:47 +00:00 committed by Gerrit Code Review
commit 6ac7873c39
10 changed files with 275 additions and 0 deletions

28
devstack/README.rst Normal file
View File

@ -0,0 +1,28 @@
===============================
Enabling Slogging in DevStack
===============================
1. Download Devstack::
git clone https://git.openstack.org/openstack-dev/devstack
cd devstack
2. Add this repo as an external repository in ``local.conf`` file::
[[local|localrc]]
enable_plugin slogging https://git.openstack.org/openstack/slogging
To use stable branches, make sure devstack is on that branch, and specify
the branch name to enable_plugin, for example::
[[local|localrc]]
enable_plugin slogging https://git.openstack.org/openstack/slogging <refspec>
In case triggered by cron, set environ variable ``SLOGGING_CRON``::
[[local|localrc]]
SLOGGING_CRON=True
3. Run ``stack.sh``::
./stack.sh

View File

@ -0,0 +1 @@
5 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-uploader %SLOGGING_BASE%/etc/log-processor.conf access

View File

@ -0,0 +1 @@
5 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-container-stats-logger %SLOGGING_BASE%/etc/log-processor.conf

View File

@ -0,0 +1 @@
0 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-account-stats-logger %SLOGGING_BASE%/etc/log-processor.conf

View File

@ -0,0 +1,2 @@
10 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-uploader %SLOGGING_BASE%/etc/log-processor.conf stats
15 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-uploader %SLOGGING_BASE%/etc/log-processor.conf container-stats

View File

@ -0,0 +1 @@
30 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-stats-collector %SLOGGING_BASE%/etc/log-processor.conf

View File

@ -0,0 +1,50 @@
[DEFAULT]
swift_account = %TEMPACCOUNT%
user = %STACK_USER%
log_level = DEBUG
log_facility = LOG_LOCAL1
[log-processor]
container_name = log_processing_data
proxy_server_conf = /etc/swift/proxy-server.conf
lookback_hours = 120
lockback_window = 120
format_type = json
[log-processor-access]
container_name = log_data
log_dir = %DEST%/data/swift/logs/hourly/
source_filename_pattern = ^
(?P<year>[0-9]{4})
(?P<month>[0-1][0-9])
(?P<day>[0-3][0-9])
(?P<hour>[0-2][0-9])
.*$
class_path = slogging.access_processor.AccessLogProcessor
unlink_log = True
new_log_cutoff = 30
[log-processor-stats]
container_name = account_stats
log_dir = %DEST%/data/swift/logs/stats/
class_path = slogging.stats_processor.StatsLogProcessor
log_name = stats-%Y%m%d%H_
stats_type = account
account_server_data_dir = accounts
devices = %DEST%/data/swift/1
mount_check = false
unlink_log = True
new_log_cutoff = 30
[log-processor-container-stats]
container_name = container_stats
log_dir = %DEST%/data/swift/logs/stats/
class_path = slogging.stats_processor.StatsLogProcessor
log_name = container-stats-%Y%m%d%H_
stats_type = container
container_server_data_dir = containers
processable = false
devices = %DEST%/data/swift/1
mount_check = false
unlink_log = True
new_log_cutoff = 30

125
devstack/lib/slogging Normal file
View File

@ -0,0 +1,125 @@
#!/bin/bash
#
# lib/slogging
# Functions to control the configuration and operation of the slogging plugin
# Dependencies:
#
# - Swift service has already installed into $DEST/swift with below in local.conf
# - ``s-proxy``, ``s-container``, ``s-object``, ``s-object``
# - ``functions`` file
# - ``DEST``, ``$STACK_USER``must be defined
# - ``lib/swift`` file
# - ``SWIFT_CONFIG_PROXY_SERVER`` must be defined
# ``stack.sh`` file
# - ``echo_summary`` function must be defined
#
# ``stack.sh`` calls the entry points in this order:
#
# - install_slogging
# - configure_slogging
# - verify_swift
# - metering-sample_slogging
function _get_token_and_endpoint {
curl -v -H "X-Storage-User:${SLOGGING['tempuser']}" \
-H "X-Storage-Pass:${SLOGGING['temppass']}" \
${SLOGGING['auth_url']} 2>&1 | \
egrep 'X-Storage-Url:|X-Auth-Token:' | \
sed -e 's|: |=|g' -e 's|<||g' -e 's|\-|_|g' -e 's|\r||g'
}
function _exec_swift_api {
for envvar in $(set | awk -F'=' '/^OS_/{print $1}'); do
unset ${envvar}
done
swift -A ${SLOGGING['auth_url']} \
-U ${SLOGGING['tempuser']} \
-K ${SLOGGING['temppass']} stat
}
function install_slogging {
pushd $SLOGGING_BASE && {
sudo python setup.py install
} && popd
}
function configure_slogging {
local saio_base="$DEST/swift/doc/saio"
local rsyslog_conf="rsyslog.d/10-swift.conf"
local swift_log_base="$DEST/data/swift"
# Set config
if [ ! -f "/etc/$rsyslog_conf" ]; then
sudo cp $saio_base/$rsyslog_conf /etc/$rsyslog_conf
fi
sudo sed -e "
s|/var/log/swift|$swift_log_base/logs|g;
s|#\$templat|\$templat|g;
s|#local1\.|local1.|g;
" -i /etc/$rsyslog_conf
sudo sed -e '/PrivDropToGroup/c $PrivDropToGroup adm' /etc/rsyslog.conf
# Set directory
sudo mkdir -p $swift_log_base/logs/hourly
safe_chown -R $STACK_USER.adm $swift_log_base
safe_chmod 775 $swift_log_base/logs $swift_log_base/logs/hourly
add_user_to_group syslog adm
add_user_to_group $STACK_USER adm
# Reflect daemon
restart_service rsyslog
# Generate /etc/swift/log-processor.conf
local log_processor=$SLOGGING_BASE/etc/log-processor.conf
sudo cp $SLOGGING_BASE/devstack/files/log-processor.conf $log_processor
# set device
iniset $log_processor log-processor-container-stats devices $swift_log_base/1
iniset $log_processor log-processor-stats devices $swift_log_base/1
# set log_dir
iniset $log_processor log-processor-container-stats log_dir $swift_log_base/logs/stats/
iniset $log_processor log-processor-stats log_dir $swift_log_base/logs/stats/
iniset $log_processor log-processor-access log_dir $swift_log_base/logs/hourly/
iniset $log_processor DEFAULT swift_account ${SLOGGING['tempaccount']}
iniset $log_processor DEFAULT user $STACK_USER
iniset $SWIFT_CONFIG_PROXY_SERVER app:proxy-server log_facility LOG_LOCAL1
# Set cron
if [ "$SLOGGING_CRON" == "True" ]; then
sudo cp $SLOGGING_BASE/devstack/files/cron.d/swift-* /etc/cron.d/
sudo sed -e "
s|%STACK_USER%|$STACK_USER|g;
s|%SLOGGING_BASE%|$SLOGGING_BASE|g;
" -i /etc/cron.d/swift-*
else
sudo rm -f /etc/cron.d/swift-*
fi
}
function verify_swift {
# Internal proxy(identity) API
for var in $(_get_token_and_endpoint); do export $var; done
curl -v -H "X-Auth-Token: $X_Auth_Token" $X_Storage_Url
# Swift API
local retry=3
local interval=5
for i in {1..$retry}; do
( _exec_swift_api ) && return 0
sleep $interval
done
return -1
}
function metering-sample_slogging {
echo_summary "Swift's Function Test"
$DEST/swift/.functests
}

57
devstack/plugin.sh Normal file
View File

@ -0,0 +1,57 @@
# plugin.sh - Devstack plugin.sh script to install and configure slogging with swift settings
# Save trace setting
XTRACE=$(set +o | grep xtrace)
set -o xtrace
echo_summary "slogging's plugin.sh was called..."
source $DEST/slogging/devstack/lib/slogging
# check for service enabled
if is_plugin_enabled slogging; then
if [[ "$1" == "stack" && "$2" == "pre-install" ]]; then
# Set up system services # apt-get install, pip insall
#echo_summary "Configuring system services Slogging"
#install_package cowsay
:
elif [[ "$1" == "stack" && "$2" == "install" ]]; then
# Perform installation of service source # python setup.py install
#echo_summary "Installing Slogging"
#install_slogging
:
elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
# Configure after the other layer 1 and 2 services have been configured
echo_summary "Configuring Slogging" # memcached, rsyslog, ... etc
configure_slogging
elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
# Initialize and start the
echo_summary "Verifying Swift nomally installed"
verify_swift
elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
# Unit & Function test for slogging to get swift log
echo_summary "Metering Sample with Swift"
metering-sample_slogging
fi
if [[ "$1" == "unstack" ]]; then
# Shut down template services
# no-op
:
fi
if [[ "$1" == "clean" ]]; then
# Remove state and transient data
# Remember clean.sh first calls unstack.sh
# no-op
:
fi
fi
# Restore
$XTRACE

9
devstack/settings Normal file
View File

@ -0,0 +1,9 @@
# settings file for sloggin plugin
declare -A -g SLOGGING
SLOGGING['tempuser']="test:tester"
SLOGGING['temppass']="testing"
SLOGGING['tempaccount']="TEMPAUTH_test"
SLOGGING['auth_url']="http://127.0.0.1:8080/auth/v1.0"
SLOGGING_BASE=$DEST/slogging
enable_service s-proxy s-object s-container s-account