openstack
/
monasca-log-api
Code Issues Proposed changes

Adjust monasca-log-api devstack to Xenial 

Following change allows to run monasca-log-api
devstack plugin on the Xenial environment.
That was implemented with the help of:
* start/stop of processes handles through run_process
  and stop_process & no services files
* setting up environment using devstack functions
** monasca-log-api virtual env (USE_VENV)
** installing monasca-log-api dependencies from git
** saving data in keystone catalog with Openstack CLI

Additionally added method that downloads tarballs
only if they are not found in the system.

Change-Id: I08b2ddbe17b2c7899056a63a5633070ae9a2c2db
This commit is contained in:
Tomasz Trębski 2016-11-15 14:02:47 +01:00
parent 09b3ec6479
commit 900e32ad8e
25 changed files with 792 additions and 1028 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -32,3 +32,6 @@ logs/
*config*.yml
db/config.yml
.venv
# Logs from devstack directory should not be committed
devstack/*.log

56
devstack/Vagrantfile vendored
View File

@ -1,41 +1,27 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
# All Vagrant configuration is done below. The "2" in Vagrant.configure
# configures the configuration version (we support older styles for
# backwards compatibility). Please don't change it unless you know what
# you're doing.
require 'vagrant.rb'
Vagrant.configure(2) do |config|
if Vagrant.has_plugin?("vagrant-cachier")
config.cache.scope = :box
config.cache.scope = :box if Vagrant.has_plugin?("vagrant-cachier")
config.timezone.value = :host if Vagrant.has_plugin?('vagrant-timezone')
if Vagrant.has_plugin?('vagrant-proxyconf')
config.proxy.http = ENV['http_proxy'] if ENV['http_proxy']
config.proxy.https = ENV['https_proxy'] if ENV['https_proxy']
if ENV['no_proxy']
local_no_proxy = ",192.168.10.6,10.0.2.15"
config.proxy.no_proxy = ENV['no_proxy'] + local_no_proxy
end
end
# Handle local proxy settings
if Vagrant.has_plugin?("vagrant-proxyconf")
if ENV["http_proxy"]
config.proxy.http = ENV["http_proxy"]
end
if ENV["https_proxy"]
config.proxy.https = ENV["https_proxy"]
end
if ENV["no_proxy"]
config.proxy.no_proxy = ENV["no_proxy"] + ',192.168.10.6,10.0.2.15'
end
end
config.ssh.forward_agent = true
config.vm.hostname = "devstack"
config.vm.box = "ubuntu/trusty64"
# config.vm.box_check_update = false
config.vm.box = "ubuntu/xenial64"
config.vm.network "private_network",ip:"192.168.10.6"
config.vm.synced_folder "~/", "/vagrant_home"
config.vm.provider "virtualbox" do |vb|
vb.gui = false
vb.memory = "12800"
vb.cpus = 4
# vb.customize ["modifyvm", :id, "--cpuexecutioncap", "50"]
end
config.vm.provision "shell", privileged: false, inline: <<-SHELL
@ -70,16 +56,20 @@ Vagrant.configure(2) do |config|
sudo cp ./maven_proxy_settings.xml /root/.m2/settings.xml
fi
git clone https://git.openstack.org/openstack-dev/devstack -b master
sudo pip install numpy
sudo pip install python-monascaclient
git clone https://git.openstack.org/openstack-dev/devstack --branch master --depth 1
cd devstack
echo '[[local|localrc]]
GIT_DEPTH=1
DEST=/opt/stack
USE_VENV=False
SERVICE_HOST=192.168.10.6
HOST_IP=192.168.10.6
HOST_IP_IFACE=eth1
DATABASE_HOST=192.168.10.6
MYSQL_HOST=192.168.10.6
HOST_IP_IFACE=eth1
MYSQL_PASSWORD=secretmysql
DATABASE_PASSWORD=secretdatabase
RABBIT_PASSWORD=secretrabbit
@ -91,7 +81,7 @@ LOGDIR=$DEST/logs
LOG_COLOR=False
disable_all_services
enable_service rabbit mysql key tempest horizon
enable_service zookeeper rabbit mysql key tempest horizon
# The following two variables allow switching between Java and Python for the implementations
# of the Monasca API and the Monasca Persister. If these variables are not set, then the
@ -107,12 +97,12 @@ MONASCA_PERSISTER_IMPLEMENTATION_LANG=${MONASCA_PERSISTER_IMPLEMENTATION_LANG:-p
# Uncomment one of the following two lines to choose either InfluxDB or Vertica.
# MONASCA_METRICS_DB=${MONASCA_METRICS_DB:-vertica}
# MONASCA_METRICS_DB=${MONASCA_METRICS_DB:-cassandra}
MONASCA_METRICS_DB=${MONASCA_METRICS_DB:-influxdb}
# Uncomment one of the following lines and modify accordingly to enable the Monasca DevStack Plugin
enable_plugin monasca-api https://git.openstack.org/openstack/monasca-api
# enable_plugin monasca-api file:///vagrant_home/Documents/repos/openstack/monasca-api.vertica
enable_plugin monasca-log-api https://github.com/openstack/monasca-log-api.git
enable_plugin monasca-log-api https://git.openstack.org/openstack/monasca-log-api.git
' > local.conf
./stack.sh

28
devstack/files/elasticsearch/elasticsearch.conf
View File

@ -1,28 +0,0 @@
#
# Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
description "elasticsearch"
start on runlevel [2345]
stop on runlevel [!2345]
respawn
setgid elastic
setuid elastic
exec /opt/elasticsearch/bin/elasticsearch

14
devstack/files/elasticsearch/elasticsearch.yml
View File

@ -136,10 +136,10 @@ node.data: true
#################################### Paths ####################################
# Path to directory where to store index data allocated for this node.
path.data: /var/data/elasticsearch
path.data: %ES_DATA_DIR%
# Path to log files:
path.logs: /var/log/elasticsearch
path.logs: %ES_LOG_DIR%
# Path to where plugins are installed:
#path.plugins: /path/to/plugins
@ -183,14 +183,14 @@ path.logs: /var/log/elasticsearch
# try the next port).
# Set the bind address specifically (IPv4 or IPv6):
network.bind_host: 0.0.0.0
network.bind_host: %ES_SERVICE_BIND_HOST%
# Set the address other nodes will use to communicate with this node. If not
# set, it is automatically derived. It must point to an actual IP address.
network.publish_host: 127.0.0.1
network.publish_host: %ES_SERVICE_PUBLISH_HOST%
# Set a custom port for the node to node communication (9300 by default):
transport.tcp.port: 9300
transport.tcp.port: %ES_SERVICE_PUBLISH_PORT%
# Enable compression for all communication between nodes (disabled by default):
#
@ -198,7 +198,7 @@ transport.tcp.port: 9300
# Set a custom port to listen for HTTP traffic:
#
http.port: 9200
http.port: %ES_SERVICE_BIND_PORT%
# Set a custom allowed content length:
#

110
devstack/files/keystone/create_monasca_log_service.py
View File

@ -1,110 +0,0 @@
#
# Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import print_function
from keystoneclient.v2_0 import client
import sys
def get_token(url, cacert, username, password, tenant_name):
if not username or not password:
print('If token is not given, keystone_admin and keystone_admin_password must be given', file=sys.stderr)
return False
if not tenant_name:
print('If token is not given, keystone_admin_project must be given', file=sys.stderr)
return False
kwargs = {
'username': username,
'password': password,
'tenant_name': tenant_name,
'auth_url': url,
'cacert': cacert
}
key = client.Client(**kwargs)
token = key.auth_token
return token
def add_service_endpoint(key, name, description, type, url, region):
"""Add the Monasca service to the catalog with the specified endpoint, if it doesn't yet exist."""
service_names = {service.name: service.id for service in key.services.list()}
if name in service_names.keys():
service_id = service_names[name]
else:
service = key.services.create(name=name, service_type=type, description=description)
print("Created service '{}' of type '{}'".format(name, type))
service_id = service.id
for endpoint in key.endpoints.list():
if endpoint.service_id == service_id:
if endpoint.publicurl == url and endpoint.adminurl == url and endpoint.internalurl == url:
return True
else:
key.endpoints.delete(endpoint.id)
key.endpoints.create(region=region, service_id=service_id, publicurl=url, adminurl=url, internalurl=url)
print("Added service endpoint '{}' at '{}'".format(name, url))
return True
def add_monasca_service():
return True
def main(argv):
""" Get token if needed and then call methods to add tenants, users and roles """
service_host = argv[0]
url = 'http://' + service_host + ':35357/v2.0'
token = None
cacert = None
if not token:
username = argv[1]
password = argv[2]
tenant_name = argv[3]
token = get_token(url, cacert, username, password, tenant_name)
key = client.Client(token=token, endpoint=url, cacert=cacert)
monasca_log_url = 'http://' + service_host + ':5607/v3.0'
if not add_service_endpoint(key, 'logs', 'Monasca log service', 'logs', monasca_log_url, 'RegionOne'):
return 1
monasca_log_v2_url = 'http://' + service_host + ':5607/v2.0'
if not add_service_endpoint(key, 'logs_v2', 'Monasca log v2 service',
'logs_v2', monasca_log_v2_url, 'RegionOne'):
return 1
monasca_log_search_url = 'http://' + service_host + ':5601/'
if not add_service_endpoint(key, 'logs-search',
'Monasca log search service', 'logs-search', monasca_log_search_url,
'RegionOne'):
return 1
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))

29
devstack/files/kibana/kibana.conf
View File

@ -1,29 +0,0 @@
#
# Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
description "Kibana"
start on runlevel [2345]
stop on runlevel [!2345]
respawn
#setgid kibana
#setuid kibana
#env NODE_OPTIONS="--harmony"
exec /opt/kibana/bin/kibana --config /opt/kibana/config/kibana.yml

14
devstack/files/kibana/kibana.yml
View File

@ -1,15 +1,15 @@
# Kibana is served by a back end server. This controls which port to use.
server.port: 5601
server.port: %KIBANA_SERVICE_PORT%
# The host to bind the server to.
server.host: 127.0.0.1
server.host: %KIBANA_SERVICE_HOST%
# If you are running kibana behind a proxy, and want to mount it at a path,
# specify that path here. The basePath can't end in a slash.
server.basePath: /dashboard/monitoring/logs_proxy
# The Elasticsearch instance to use for all your queries.
elasticsearch.url: http://localhost:9200
elasticsearch.url: http://%ES_SERVICE_BIND_HOST%:%ES_SERVICE_BIND_PORT%
# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false,
# then the host you use to connect to *this* Kibana instance will be sent.
@ -49,7 +49,6 @@ kibana.defaultAppId: "discover"
# request_timeout setting
elasticsearch.pingTimeout: 1500
# Time in milliseconds to wait for responses from the back end or elasticsearch.
# This must be > 0
elasticsearch.requestTimeout: 300000
@ -64,9 +63,6 @@ elasticsearch.startupTimeout: 5000
# Set the path to where you would like the process id file to be created.
# pid.file: /var/run/kibana.pid
# If you would like to send the log output to a file you can set the path below.
logging.dest: /var/log/kibana/kibana.log
# Set this to true to suppress all logging output.
logging.silent: false
# Set this to true to suppress all logging output except for error messages.
@ -75,8 +71,8 @@ logging.quiet: false
logging.verbose: true
# fts-keystone configuration
fts-keystone.port: 35357
fts-keystone.url: http://127.0.0.1
fts-keystone.port: %KEYSTONE_AUTH_PORT%
fts-keystone.url: %KEYSTONE_AUTH_PROTOCOL%://%KEYSTONE_AUTH_HOST%
fts-keystone.enabled: True
optimize.useBundleCache: False

5
devstack/files/monasca-log-agent/agent.conf
View File

@ -32,10 +32,11 @@ filter {
}
}
# TODO(trebskit) should use own user for log-agent
output {
monasca_log_api {
monasca_log_api_url => "http://127.0.0.1:5607/v3.0"
keystone_api_url => "http://127.0.0.1:35357/v3"
monasca_log_api_url => "%MONASCA_LOG_API_URI_V3%"
keystone_api_url => "%KEYSTONE_AUTH_URI_V3%"
project_name => "mini-mon"
username => "monasca-agent"
password => "password"

25
devstack/files/monasca-log-agent/monasca-log-agent.conf
View File

@ -1,25 +0,0 @@
#
# Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
description "monasca-log-agent"
start on runlevel [2345]
stop on runlevel [!2345]
respawn
exec sudo /opt/logstash/bin/logstash -f /etc/monasca/monasca-log-agent/agent.conf -l /var/log/monasca/monasca-log-agent/monasca-log-agent.log

52
devstack/files/monasca-log-api/log-api-config.conf
View File

@ -1,52 +0,0 @@
#
# Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
[DEFAULT]
log_config_append=/etc/monasca/log-api-logging.conf
[service]
region = useast
max_log_size = 1048576
[log_publisher]
topics = log
kafka_url = 127.0.0.1:9092
[keystone_authtoken]
identity_uri = http://127.0.0.1:35357
auth_uri = http://127.0.0.1:5000
auth_version = 3
admin_password = secretadmin
admin_user = admin
admin_tenant_name = admin
insecure = True
memcached_servers = 127.0.0.1:11211
[kafka_healthcheck]
kafka_url = 127.0.0.1:9092
kafka_topics = log
[roles_middleware]
path = /v2.0/log,/v3.0/logs
default_roles = user,domainuser,domainadmin,monasca-user
agent_roles = monasca-agent
[dispatcher]
logs = monasca_log_api.reference.v2.logs:Logs
logs_v3 = monasca_log_api.reference.v3.logs:Logs
versions = monasca_log_api.reference.versions:Versions
healthchecks = monasca_log_api.reference.healthchecks:HealthChecks

43
devstack/files/monasca-log-api/log-api-config.ini
View File

@ -1,43 +0,0 @@
#
# Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
[DEFAULT]
name = monasca_log_api
[pipeline:main]
# Add validator in the pipeline so the metrics messages can be validated.
pipeline = auth roles api
[app:api]
paste.app_factory = monasca_log_api.server:launch
[filter:auth]
paste.filter_factory = monasca_log_api.healthcheck.keystone_protocol:filter_factory
[filter:roles]
paste.filter_factory = monasca_log_api.middleware.role_middleware:RoleMiddleware.factory
[server:main]
use = egg:gunicorn#main
host = 127.0.0.1
port = 5607
workers = 1
proc_name = monasca_log_api
user = mon-log-api
group = monasca
loglevel = DEBUG
chdir = /opt/monasca-log-api

36
devstack/files/monasca-log-api/log-api-logging.conf
View File

@ -1,36 +0,0 @@
[loggers]
keys = root, kafka
[handlers]
keys = console, file
[formatters]
keys = generic
[logger_root]
level = DEBUG
formatter = default
handlers = console, file
[logger_kafka]
qualname = kafka
level = DEBUG
formatter = default
handlers = console, file
propagate = 0
[handler_console]
class = logging.StreamHandler
args = (sys.stderr,)
level = DEBUG
formatter = generic
[handler_file]
class = logging.handlers.RotatingFileHandler
level = DEBUG
formatter = generic
# store up to 5*100MB of logs
args = ('/var/log/monasca/log-api/monasca-log-api.log', 'a', 104857600, 5)
[formatter_generic]
format = %(asctime)s %(levelname)s [%(name)s][%(threadName)s] %(message)s

27
devstack/files/monasca-log-api/monasca-log-api.conf
View File

@ -1,27 +0,0 @@
#
# Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Startup script for the Monasca LOG API
description "Monasca LOG API Python app"
start on runlevel [2345]
console log
respawn
setgid monasca
setuid mon-log-api
exec /opt/monasca-log-api/bin/gunicorn -n monasca-log-api -k eventlet --worker-connections=2000 --backlog=1000 --paste /etc/monasca/log-api-config.ini -w 9

2
devstack/files/monasca-log-metrics/log-metrics.conf
View File

@ -73,7 +73,7 @@ filter {
output {
kafka {
bootstrap_servers => "127.0.0.1:9092"
bootstrap_servers => "%KAFKA_SERVICE_HOST%:%KAFKA_SERVICE_PORT%"
topic_id => "metrics"
client_id => "monasca_log_metrics"
compression_type => "none"

28
devstack/files/monasca-log-metrics/monasca-log-metrics.conf
View File

@ -1,28 +0,0 @@
#
# Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
description "monasca-log-metrics"
start on runlevel [2345]
stop on runlevel [!2345]
respawn
setgid monasca
setuid mon-log-metrics
exec /opt/logstash/bin/logstash -f /etc/monasca/log/log-metrics.conf -l /var/log/monasca/monasca-log-metrics/log-metrics.log --verbose

28
devstack/files/monasca-log-persister/monasca-log-persister.conf
View File

@ -1,28 +0,0 @@
#
# Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
description "monasca-log-persister"
start on runlevel [2345]
stop on runlevel [!2345]
respawn
setgid monasca
setuid mon-persister
exec /opt/logstash/bin/logstash -f /etc/monasca/log/persister.conf -l /var/log/monasca/monasca-log-persister/logs-persister.log --verbose

2
devstack/files/monasca-log-persister/persister.conf
View File

@ -65,7 +65,7 @@ output {
elasticsearch {
index => "%{tenant}-%{index_date}"
document_type => "log"
hosts => ["127.0.0.1"]
hosts => ["%ES_SERVICE_BIND_HOST%"]
flush_size => 500
}
}

28
devstack/files/monasca-log-transformer/monasca-log-transformer.conf
View File

@ -1,28 +0,0 @@
#
# Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
description "monasca-log-transformer"
start on runlevel [2345]
stop on runlevel [!2345]
respawn
setgid monasca
setuid mon-transformer
exec /opt/logstash/bin/logstash -f /etc/monasca/log/transformer.conf -l /var/log/monasca/monasca-log-transformer/log-transformer.log --verbose

2
devstack/files/monasca-log-transformer/transformer.conf
View File

@ -81,7 +81,7 @@ filter {
output {
kafka {
bootstrap_servers => "127.0.0.1:9092"
bootstrap_servers => "%KAFKA_SERVICE_HOST%:%KAFKA_SERVICE_PORT%"
topic_id => "transformed-log"
}
}

1101
devstack/plugin.sh
View File

File diff suppressed because it is too large Load Diff

68
devstack/settings
View File

@ -14,8 +14,74 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
enable_service monasca_log
# services
enable_service zookeeper # devstack/lib/dlm
enable_service kibana # devstack/monasca-api/kafka
enable_service elasticsearch
enable_service monasca-log
enable_service monasca-log-api
enable_service monasca-log-persister
enable_service monasca-log-transformer
enable_service monasca-log-metrics
enable_service monasca-log-agent
# libraries/frameworks/projects monasca-log requires to install itself
KIBANA_VERSION=${KIBANA_VERSION:-4.4.0-linux-x64}
LOGSTASH_VERSION=${LOGSTASH_VERSION:-2.2.0}
ELASTICSEARCH_VERSION=${ELASTICSEARCH_VERSION:-2.2.0}
NODE_JS_VERSION=${NODE_JS_VERSION:-"4.0.0"}
NVM_VERSION=${NVM_VERSION:-"0.32.1"}
# repository settings
MONASCA_LOG_API_REPO=${MONASCA_LOG_API_REPO:-${GIT_BASE}/openstack/monasca-log-api.git}
MONASCA_LOG_API_BRANCH=${MONASCA_LOG_API_BRANCH:-master}
MONASCA_LOG_API_DIR=${DEST}/monasca-log-api
MONASCA_COMMON_REPO=${MONASCA_COMMON_REPO:-${GIT_BASE}/openstack/monasca-common.git}
MONASCA_COMMON_BRANCH=${MONASCA_COMMON_BRANCH:-master}
MONASCA_COMMON_DIR=${DEST}/monasca-common
MONASCA_STATSD_REPO=${MONASCA_STATSD_REPO:-${GIT_BASE}/openstack/monasca-statsd.git}
MONASCA_STATSD_BRANCH=${MONASCA_STATSD_BRANCH:-master}
MONASCA_STATSD_DIR=${DEST}/monasca-statsd
FTS_KEYSTONE_REPO=${FTS_KEYSTONE_REPO:-"https://github.com/FujitsuEnablingSoftwareTechnologyGmbH/fts-keystone.git"}
FTS_KEYSTONE_BRANCH=${FTS_KEYSTONE_BRANCH:-master}
FTS_KEYSTONE_DIR=${DEST}/fts-keystone
GITREPO["monasca-common"]=${MONASCA_COMMON_REPO}
GITBRANCH["monasca-common"]=${MONASCA_COMMON_BRANCH}
GITDIR["monasca-common"]=${MONASCA_COMMON_DIR}
GITREPO["monasca-statsd"]=${MONASCA_STATSD_REPO}
GITBRANCH["monasca-statsd"]=${MONASCA_STATSD_BRANCH}
GITDIR["monasca-statsd"]=${MONASCA_STATSD_DIR}
LIBS_FROM_GIT="${LIBS_FROM_GIT:-""},monasca-common,monasca-statsd"
# configuration bits
MONASCA_LOG_API_CONF_DIR=${MONASCA_LOG_API_CONF_DIR:-/etc/monasca}
MONASCA_LOG_API_CONF=${MONASCA_LOG_API_CONF:-$MONASCA_LOG_API_CONF_DIR/log-api-config.conf}
MONASCA_LOG_API_PASTE_INI=${MONASCA_LOG_API_PASTE_INI:-$MONASCA_LOG_API_CONF_DIR/log-api-config.ini}
MONASCA_LOG_API_LOGGING_CONF=${MONASCA_LOG_API_LOGGING_CONF:-$MONASCA_LOG_API_CONF_DIR/log-api-logging.conf}
MONASCA_LOG_API_CACHE_DIR=${MONASCA_LOG_API_CACHE_DIR:-/var/cache/monasca-log-api}
# public facing bits
MONASCA_LOG_API_SERVICE_HOST=${MONASCA_LOG_API_SERVICE_HOST:-${SERVICE_HOST}}
MONASCA_LOG_API_SERVICE_PORT=${MONASCA_LOG_API_SERVICE_PORT:-5607}
MONASCA_LOG_API_SERVICE_PROTOCOL="http" # right now we do not have SSL support here
ES_SERVICE_BIND_HOST=${ES_SERVICE_BIND_HOST:-${SERVICE_HOST}}
ES_SERVICE_BIND_PORT=${ES_SERVICE_BIND_PORT:-9200}
ES_SERVICE_PUBLISH_HOST=${ES_SERVICE_PUBLISH_HOST:-${SERVICE_HOST}}
ES_SERVICE_PUBLISH_PORT=${ES_SERVICE_PUBLISH_PORT:-9300}
KIBANA_SERVICE_HOST=${KIBANA_SERVICE_HOST:-${SERVICE_HOST}}
KIBANA_SERVICE_PORT=${KIBANA_SERVICE_PORT:-5601}
KIBANA_SERVER_BASE_PATH=${KIBANA_SERVER_BASE_PATH:-"/dashboard/monitoring/logs_proxy"}
KAFKA_SERVICE_HOST=${KAFKA_SERVICE_HOST:-${SERVICE_HOST}}
KAFKA_SERVICE_PORT=${KAFKA_SERVICE_PORT:-9092}
DOWNLOAD_FILE_TIMEOUT=${DOWNLOAD_FILE_TIMEOUT:-300}

68
etc/monasca/log-api-config.conf
View File

@ -1,9 +1,25 @@
#
# Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# note(trebskit) all URLs points to default monasca-log-api devstack setup
[DEFAULT]
# logging, make sure that the user under whom the server runs has permission
# to write to the directory.
log_config_append=/etc/monasca/log-api-logging.conf
# Dispatchers to be loaded to serve restful APIs
[dispatcher]
logs = monasca_log_api.reference.v2.logs:Logs
logs_v3 = monasca_log_api.reference.v3.logs:Logs
@ -16,48 +32,30 @@ statsd_port = 8125
statsd_buffer = 50
[service]
region = region-one
max_log_size = 1048576
region = 'region-one'
[log_publisher]
topics = 'logs'
# Uncomment for DevStack environment
kafka_url = '192.168.10.6:9092'
# Uncomment for monasca-vagrant environment
#kafka_url = '192.168.10.4:9092'
#Message max size that can be sent to Kafka
topics = log
kafka_url = 192.168.10.6:9092
max_message_size = 1048576
[keystone_authtoken]
# Uncomment for DevStack environment
identity_uri = http://192.168.10.6:35357
auth_uri = http://192.168.10.6:5000
admin_password = secretadmin
# Uncomment for monaasca-vagrant environment
# identity_uri = http://192.168.10.5:35357
# auth_uri = http://192.168.10.5:5000
# admin_password = admin
auth_uri = http://127.0.0.1:5000
auth_url = http://127.0.0.1:5000
identity_uri = http://127.0.0.1:35357
admin_user = admin
admin_password = secretadmin
admin_tenant_name = admin
cafile =
certfile =
keyfile =
insecure = false
# memcached_servers = 127.0.0.1:11211
# token_cache_time=300
insecure = True
token_cache_time = 300
memcached_servers = 127.0.0.1:11211
[kafka_healthcheck]
kafka_url = localhost:8900
kafka_url = 192.168.10.6:9092
kafka_topics = log
[roles_middleware]
path = /v2.0/log
path = /v3.0/logs
default_roles = user, domainuser, domainadmin, monasca-user
agent_roles = monasca-agent, admin
path = /v2.0/log,/v3.0/logs
default_roles = user,domainuser,domainadmin,monasca-user
agent_roles = monasca-agent

23
etc/monasca/log-api-config.ini
View File

@ -1,3 +1,20 @@
#
# Copyright 2016 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
[DEFAULT]
name = monasca_log_api
@ -17,5 +34,9 @@ paste.filter_factory = monasca_log_api.middleware.role_middleware:RoleMiddleware
use = egg:gunicorn#main
host = 127.0.0.1
port = 5607
workers = 1
workers = 9
worker-connections = 2000
backlog = 1000
proc_name = monasca_log_api
loglevel = DEBUG
chdir = /opt/monasca-log-api

4
etc/monasca/log-api-logging.conf
View File

@ -30,7 +30,7 @@ class = logging.handlers.RotatingFileHandler
level = DEBUG
formatter = generic
# store up to 5*100MB of logs
args = ('/var/log/monasca/log-api/monasca-log-api.log', 'a', 104857600, 5)
args = ('monasca-log-api.log', 'a', 104857600, 5)
[formatter_generic]
format = %(asctime)s %(levelname)s [%(name)s][%(threadName)s] %(message)s
format = %(asctime)s %(levelname)s [%(name)s][%(threadName)s] %(message)s

24
monasca_log_api_tempest/clients.py
View File

@ -22,17 +22,19 @@ from monasca_log_api_tempest.services import log_search_client
class Manager(clients.Manager):
def __init__(self, credentials=None, service=None):
super(Manager, self).__init__(credentials, service)
self.log_api_clients = {}
self.log_api_clients["v2"] = log_api_v2_client.LogApiV2Client(
self.auth_provider,
'logs_v2',
None
)
self.log_api_clients["v3"] = log_api_v3_client.LogApiV3Client(
self.auth_provider,
'logs',
None
)
self.log_api_clients = {
"v2": log_api_v2_client.LogApiV2Client(
self.auth_provider,
'logs_v2',
None
),
"v3": log_api_v3_client.LogApiV3Client(
self.auth_provider,
'logs',
None
)
}
self.log_search_client = log_search_client.LogsSearchClient(
self.auth_provider,
'logs-search',