retrieve project id to ignore from keystone

We currently allows only project uuid, but this is a pain for deployer.
Also the default is a project name which doesn't work...

This change queries keystone to retrieve project ids when the
ignore_projects list are names.

By default, if auth_type is not set, we keep the previous behavior to
not query keystone.

This change also vendors keystoneauth1.loading.adapter that wasn't
exists in newton version of keystoneauth1.

(cherry picked from commit e2bf485044)

Fix default service project to "service"

Both devstack and TripleO uses "service" by default.
Nothing uses "services".

(cherry picked from commit fbd048f4c0)

Change-Id: I270d080d3e65eb6b0cd823498a4dd37389c49221
This commit is contained in:
Mehdi Abaakouk 2017-05-13 13:37:33 +02:00
parent 9f6a8e70a5
commit 4252674cbd
6 changed files with 770 additions and 5 deletions

View File

@ -0,0 +1,131 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystoneauth1 import adapter
from keystoneauth1.loading import _utils
from keystoneauth1.loading import base
__all__ = ('register_argparse_arguments',
'register_service_argparse_arguments',
'register_conf_options',
'load_from_conf_options',
'get_conf_options')
class Adapter(base.BaseLoader):
@property
def plugin_class(self):
return adapter.Adapter
def get_options(self):
return []
@staticmethod
def get_conf_options():
"""Get oslo_config options that are needed for a :py:class:`.Adapter`.
These may be useful without being registered for config file generation
or to manipulate the options before registering them yourself.
The options that are set are:
:service_type: The default service_type for URL discovery.
:service_name: The default service_name for URL discovery.
:interface: The default interface for URL discovery.
:region_name: The default region_name for URL discovery.
:endpoint_override: Always use this endpoint URL for requests
for this client.
:returns: A list of oslo_config options.
"""
cfg = _utils.get_oslo_config()
return [cfg.StrOpt('service-type',
help='The default service_type for endpoint URL '
'discovery.'),
cfg.StrOpt('service-name',
help='The default service_name for endpoint URL '
'discovery.'),
cfg.StrOpt('interface',
help='The default interface for endpoint URL '
'discovery.'),
cfg.StrOpt('region-name',
help='The default region_name for endpoint URL '
'discovery.'),
cfg.StrOpt('endpoint-override',
help='Always use this endpoint URL for requests '
'for this client.'),
]
def register_conf_options(self, conf, group):
"""Register the oslo_config options that are needed for an Adapter.
The options that are set are:
:service_type: The default service_type for URL discovery.
:service_name: The default service_name for URL discovery.
:interface: The default interface for URL discovery.
:region_name: The default region_name for URL discovery.
:endpoint_override: Always use this endpoint URL for requests
for this client.
:param oslo_config.Cfg conf: config object to register with.
:param string group: The ini group to register options in.
:returns: The list of options that was registered.
"""
opts = self.get_conf_options()
conf.register_group(_utils.get_oslo_config().OptGroup(group))
conf.register_opts(opts, group=group)
return opts
def load_from_conf_options(self, conf, group, **kwargs):
"""Create an Adapter object from an oslo_config object.
The options must have been previously registered with
register_conf_options.
:param oslo_config.Cfg conf: config object to register with.
:param string group: The ini group to register options in.
:param dict kwargs: Additional parameters to pass to Adapter
construction.
:returns: A new Adapter object.
:rtype: :py:class:`.Adapter`
"""
c = conf[group]
kwargs.setdefault('service_type', c.service_type)
kwargs.setdefault('service_name', c.service_name)
kwargs.setdefault('interface', c.interface)
kwargs.setdefault('region_name', c.region_name)
kwargs.setdefault('endpoint_override', c.endpoint_override)
return self.load_from_options(**kwargs)
def register_argparse_arguments(*args, **kwargs):
return adapter.register_adapter_argparse_arguments(*args, **kwargs)
def register_service_argparse_arguments(*args, **kwargs):
return adapter.register_service_adapter_argparse_arguments(*args, **kwargs)
def register_conf_options(*args, **kwargs):
return Adapter().register_conf_options(*args, **kwargs)
def load_from_conf_options(*args, **kwargs):
return Adapter().load_from_conf_options(*args, **kwargs)
def get_conf_options():
return Adapter.get_conf_options()

View File

@ -36,7 +36,7 @@ before "proxy-server" and add the following filter in the file:
# set topic
topic = notifications
# skip metering of requests from listed project ids
ignore_projects = <proj_uuid>, <proj_uuid2>
ignore_projects = <proj_uuid>, <proj_uuid2>, <proj_name>
# Whether to send events to messaging driver in a background thread
nonblocking_notify = False
# Queue size for sending notifications in background thread (0=unlimited).
@ -44,10 +44,25 @@ before "proxy-server" and add the following filter in the file:
send_queue_size = 1000
# Logging level control
log_level = WARNING
# All keystoneauth1 options can be set to query project name for
# ignore_projects option, here is just a example:
auth_type = password
auth_url = https://[::1]:5000
project_name = service
project_domain_name = Default
username = user
user_domain_name = Default
password = a_big_secret
interface = public
"""
import functools
import logging
from keystoneauth1 import exceptions as ksa_exc
from keystoneauth1.loading import base as ksa_base
from keystoneauth1.loading import session as ksa_session
from keystoneclient.v3 import client as ks_client
from oslo_config import cfg
import oslo_messaging
from oslo_utils import strutils
@ -62,9 +77,20 @@ import six.moves.queue as queue
import six.moves.urllib.parse as urlparse
import threading
from ceilometermiddleware import ksa_adapter
_LOG = logging.getLogger(__name__)
def list_from_csv(comma_separated_str):
if comma_separated_str:
return list(
filter(lambda x: x,
map(lambda x: x.strip(),
comma_separated_str.split(','))))
return []
def _log_and_ignore_error(fn):
@functools.wraps(fn)
def wrapper(*args, **kwargs):
@ -106,17 +132,30 @@ class InputProxy(object):
return line
class KeystoneClientLoader(ksa_adapter.Adapter):
"""Keystone client adapter loader.
Keystone client and Keystoneauth1 adapter take exactly the same options, so
it's safe to create a keystone client with keystoneauth adapter options.
"""
@property
def plugin_class(self):
return ks_client.Client
class Swift(object):
"""Swift middleware used for counting requests."""
event_queue = None
threadLock = threading.Lock()
DEFAULT_IGNORE_PROJECT_NAMES = ['service']
def __init__(self, app, conf):
self._app = app
self.ignore_projects = [
proj.strip() for proj in
conf.get('ignore_projects', 'gnocchi').split(',')]
self.ignore_projects = self._get_ignore_projects(conf)
oslo_messaging.set_transport_defaults(conf.get('control_exchange',
'swift'))
@ -157,6 +196,54 @@ class Swift(object):
_LOG.debug('Started sender thread')
Swift.threadLock.release()
def _get_ignore_projects(self, conf):
if 'auth_type' not in conf:
_LOG.info("'auth_type' is not set assuming ignore_projects are "
"only project uuid.")
return list_from_csv(conf.get('ignore_projects'))
if 'ignore_projects' in conf:
ignore_projects = list_from_csv(conf.get('ignore_projects'))
else:
ignore_projects = self.DEFAULT_IGNORE_PROJECT_NAMES
if not ignore_projects:
return []
def opt_getter(opt):
# TODO(sileht): This method does not support deprecated opt names
val = conf.get(opt.name)
if val is None:
val = conf.get(opt.dest)
return val
auth_type = conf.get('auth_type')
plugin = ksa_base.get_plugin_loader(auth_type)
auth = plugin.load_from_options_getter(opt_getter)
session = ksa_session.Session().load_from_options_getter(
opt_getter, auth=auth)
client = KeystoneClientLoader().load_from_options_getter(
opt_getter, session=session)
projects = []
for name_or_id in ignore_projects:
projects.extend(self._get_keystone_projects(client, name_or_id))
return projects
@staticmethod
def _get_keystone_projects(client, name_or_id):
try:
return [client.projects.get(name_or_id)]
except ksa_exc.NotFound:
pass
if isinstance(name_or_id, six.binary_type):
name_or_id = name_or_id.decode('utf-8', 'strict')
projects = client.projects.list(name=name_or_id)
if not projects:
_LOG.warning("fail to find project '%s' in keystone", name_or_id)
return [p.id for p in projects]
def __call__(self, env, start_response):
start_response_args = [None]
input_proxy = InputProxy(env['wsgi.input'])

View File

@ -0,0 +1,514 @@
http_interactions:
- recorded_at: '2017-05-15T07:49:52'
request:
body:
encoding: utf-8
string: |-
{
"auth": {
"tenantName": "dummy",
"passwordCredentials": {
"username": "dummy",
"password": "********"
}
}
}
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '107'
Content-Type:
- application/json
User-Agent:
- run.py keystoneauth1/2.20.0 python-requests/2.14.2 CPython/2.7.13
method: POST
uri: https://[::1]:5000/v2.0/tokens
response:
body:
encoding: null
string: |-
{
"access": {
"serviceCatalog": [
{
"type": "compute",
"endpoints_links": [],
"name": "nova",
"endpoints": [
{
"internalURL": "https://[::1]:8774/v2.1",
"adminURL": "https://[::1]:8774/v2.1",
"id": "1e879ab434b54b8abfd275feeb2ef9f3",
"region": "RegionOne",
"publicURL": "https://[::1]:8774/v2.1"
}
]
},
{
"type": "network",
"endpoints_links": [],
"name": "neutron",
"endpoints": [
{
"internalURL": "http://[::1]:9696",
"adminURL": "http://[::1]:9696",
"id": "83fcb786f646437f9a61cef72a9e43d7",
"region": "RegionOne",
"publicURL": "http://[::1]:9696"
}
]
},
{
"type": "volumev2",
"endpoints_links": [],
"name": "cinderv2",
"endpoints": [
{
"internalURL": "https://[::1]:8776/v2/ed980105f9d047e2bee738b3f261f126",
"adminURL": "https://[::1]:8776/v2/ed980105f9d047e2bee738b3f261f126",
"id": "973ef665c2ea4ec3b5c3d48932fad7a4",
"region": "RegionOne",
"publicURL": "https://[::1]:8776/v2/ed980105f9d047e2bee738b3f261f126"
}
]
},
{
"type": "volumev3",
"endpoints_links": [],
"name": "cinderv3",
"endpoints": [
{
"internalURL": "https://[::1]:8776/v3/ed980105f9d047e2bee738b3f261f126",
"adminURL": "https://[::1]:8776/v3/ed980105f9d047e2bee738b3f261f126",
"id": "0e80fe643d4d44729db99d0a5c882d1b",
"region": "RegionOne",
"publicURL": "https://[::1]:8776/v3/ed980105f9d047e2bee738b3f261f126"
}
]
},
{
"type": "image",
"endpoints_links": [],
"name": "glance",
"endpoints": [
{
"internalURL": "http://[::1]:9292",
"adminURL": "http://[::1]:9292",
"id": "7aad24b660a94254adc3546e4de4d668",
"region": "RegionOne",
"publicURL": "http://[::1]:9292"
}
]
},
{
"type": "volume",
"endpoints_links": [],
"name": "cinder",
"endpoints": [
{
"internalURL": "https://[::1]:8776/v1/ed980105f9d047e2bee738b3f261f126",
"adminURL": "https://[::1]:8776/v1/ed980105f9d047e2bee738b3f261f126",
"id": "8191ee00b695483796a9531bca70279b",
"region": "RegionOne",
"publicURL": "https://[::1]:8776/v1/ed980105f9d047e2bee738b3f261f126"
}
]
},
{
"type": "identity",
"endpoints_links": [],
"name": "keystone",
"endpoints": [
{
"internalURL": "https://[::1]:5000",
"adminURL": "https://[::1]:35357",
"id": "24ab268f1a7b47d4af493c4c74cd6130",
"region": "RegionOne",
"publicURL": "https://[::1]:5000"
}
]
}
],
"user": {
"username": "dummy",
"roles_links": [],
"id": "f18b121edda04346b86610fa23983a0e",
"roles": [
{
"name": "admin"
}
],
"name": "dummy"
},
"token": {
"issued_at": "2017-05-15T07:49:52.000000Z",
"tenant": {
"enabled": true,
"id": "ed980105f9d047e2bee738b3f261f126",
"name": "dummy",
"description": "admin tenant"
},
"audit_ids": [
"VzK7yoNFT0qlUWg5KhDuMQ"
],
"expires": "9999-12-31T23:59:59Z",
"id": "gAAAAABZGV2gZwV0SlycA_OIohX7kRAmTp84SnzsAYD5Uhey7RTzCf0NvxNRxLk5RsjRSMncSdro7eWgtMrSblZJCPl485IvHANL3E3gsxFJP9TjebqDiF4DtGhQmc4mHPB3kOBNzg3v2FrHB2hK77Cc4M7V1Pm_-nWBHxYxVNopVhrd80Y4-2c"
},
"metadata": {
"is_admin": 0,
"roles": [
"d3b61a4656d64cbbbdb0f13690e2ffe4"
]
}
}
}
headers:
Connection:
- Keep-Alive
Content-Length:
- '3183'
Content-Type:
- application/json
Date:
- Mon, 15 May 2017 07:49:51 GMT
Keep-Alive:
- timeout=3, max=100
Server:
- Apache/2.4.18 (Ubuntu)
Strict-Transport-Security:
- max-age=15768000
Vary:
- X-Auth-Token
X-Distribution:
- Ubuntu
x-openstack-request-id:
- req-84cb5714-49dc-4bab-93ba-2b66ba566c30
status:
code: 200
message: OK
url: https://[::1]:5000/v2.0/tokens
- recorded_at: '2017-05-15T07:49:53'
request:
body:
encoding: utf-8
string: ''
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- run.py keystoneauth1/2.20.0 python-requests/2.14.2 CPython/2.7.13
method: GET
uri: https://[::1]:35357/
response:
body:
encoding: null
string: |-
{
"versions": {
"values": [
{
"status": "stable",
"updated": "2016-10-06T00:00:00Z",
"id": "v3.7",
"links": [
{
"rel": "self",
"href": "https://[::1]:35357/v3/"
}
],
"media-types": [
{
"type": "application/vnd.openstack.identity-v3+json",
"base": "application/json"
}
]
},
{
"status": "deprecated",
"updated": "2016-08-04T00:00:00Z",
"id": "v2.0",
"links": [
{
"rel": "self",
"href": "https://[::1]:35357/v2.0/"
},
{
"type": "text/html",
"rel": "describedby",
"href": "http://docs.openstack.org/"
}
],
"media-types": [
{
"type": "application/vnd.openstack.identity-v2.0+json",
"base": "application/json"
}
]
}
]
}
}
headers:
Connection:
- Keep-Alive
Content-Length:
- '627'
Content-Type:
- application/json
Date:
- Mon, 15 May 2017 07:49:52 GMT
Keep-Alive:
- timeout=3, max=100
Server:
- Apache/2.4.18 (Ubuntu)
Strict-Transport-Security:
- max-age=15768000
Vary:
- X-Auth-Token
X-Distribution:
- Ubuntu
status:
code: 300
message: Multiple Choices
url: https://[::1]:35357/
- recorded_at: '2017-05-15T07:49:53'
request:
body:
encoding: utf-8
string: ''
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- python-keystoneclient
X-Auth-Token:
- gAAAAABZGV2gZwV0SlycA_OIohX7kRAmTp84SnzsAYD5Uhey7RTzCf0NvxNRxLk5RsjRSMncSdro7eWgtMrSblZJCPl485IvHANL3E3gsxFJP9TjebqDiF4DtGhQmc4mHPB3kOBNzg3v2FrHB2hK77Cc4M7V1Pm_-nWBHxYxVNopVhrd80Y4-2c
method: GET
uri: https://[::1]:35357/v3/projects/service
response:
body:
encoding: null
string: |-
{
"error": {
"code": 404,
"title": "Not Found",
"message": "Could not find project: service"
}
}
headers:
Connection:
- Keep-Alive
Content-Length:
- '93'
Content-Type:
- application/json
Date:
- Mon, 15 May 2017 07:49:53 GMT
Keep-Alive:
- timeout=3, max=99
Server:
- Apache/2.4.18 (Ubuntu)
Strict-Transport-Security:
- max-age=15768000
Vary:
- X-Auth-Token
X-Distribution:
- Ubuntu
x-openstack-request-id:
- req-6107025c-e09e-437a-90c2-61a559154d32
status:
code: 404
message: Not Found
url: https://[::1]:35357/v3/projects/service
- recorded_at: '2017-05-15T07:49:53'
request:
body:
encoding: utf-8
string: ''
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- python-keystoneclient
X-Auth-Token:
- gAAAAABZGV2gZwV0SlycA_OIohX7kRAmTp84SnzsAYD5Uhey7RTzCf0NvxNRxLk5RsjRSMncSdro7eWgtMrSblZJCPl485IvHANL3E3gsxFJP9TjebqDiF4DtGhQmc4mHPB3kOBNzg3v2FrHB2hK77Cc4M7V1Pm_-nWBHxYxVNopVhrd80Y4-2c
method: GET
uri: https://[::1]:35357/v3/projects?name=service
response:
body:
encoding: null
string: |-
{
"projects": [
{
"enabled": true,
"id": "147cc0a9263c4964926f3ee7b6ba3685",
"domain_id": "default",
"parent_id": "default",
"is_domain": false,
"name": "service",
"links": {
"self": "https://[::1]:5000/v3/projects/147cc0a9263c4964926f3ee7b6ba3685"
},
"description": "Tenant for the openstack service"
}
],
"links": {
"self": "https://[::1]:5000/v3/projects?name=service",
"next": null,
"previous": null
}
}
headers:
Connection:
- Keep-Alive
Content-Length:
- '440'
Content-Type:
- application/json
Date:
- Mon, 15 May 2017 07:49:53 GMT
Keep-Alive:
- timeout=3, max=98
Server:
- Apache/2.4.18 (Ubuntu)
Strict-Transport-Security:
- max-age=15768000
Vary:
- X-Auth-Token
X-Distribution:
- Ubuntu
x-openstack-request-id:
- req-1915b2be-f116-4831-a7c3-5ba0a32d416f
status:
code: 200
message: OK
url: https://[::1]:35357/v3/projects?name=service
- recorded_at: '2017-05-15T07:49:53'
request:
body:
encoding: utf-8
string: ''
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- python-keystoneclient
X-Auth-Token:
- gAAAAABZGV2gZwV0SlycA_OIohX7kRAmTp84SnzsAYD5Uhey7RTzCf0NvxNRxLk5RsjRSMncSdro7eWgtMrSblZJCPl485IvHANL3E3gsxFJP9TjebqDiF4DtGhQmc4mHPB3kOBNzg3v2FrHB2hK77Cc4M7V1Pm_-nWBHxYxVNopVhrd80Y4-2c
method: GET
uri: https://[::1]:35357/v3/projects/gnocchi
response:
body:
encoding: null
string: |-
{
"error": {
"code": 404,
"title": "Not Found",
"message": "Could not find project: gnocchi"
}
}
headers:
Connection:
- Keep-Alive
Content-Length:
- '92'
Content-Type:
- application/json
Date:
- Mon, 15 May 2017 07:49:53 GMT
Keep-Alive:
- timeout=3, max=97
Server:
- Apache/2.4.18 (Ubuntu)
Strict-Transport-Security:
- max-age=15768000
Vary:
- X-Auth-Token
X-Distribution:
- Ubuntu
x-openstack-request-id:
- req-b23e72d3-742e-4e10-b9a7-d1161f1eeab4
status:
code: 404
message: Not Found
url: https://[::1]:35357/v3/projects/gnocchi
- recorded_at: '2017-05-15T07:49:53'
request:
body:
encoding: utf-8
string: ''
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- python-keystoneclient
X-Auth-Token:
- gAAAAABZGV2gZwV0SlycA_OIohX7kRAmTp84SnzsAYD5Uhey7RTzCf0NvxNRxLk5RsjRSMncSdro7eWgtMrSblZJCPl485IvHANL3E3gsxFJP9TjebqDiF4DtGhQmc4mHPB3kOBNzg3v2FrHB2hK77Cc4M7V1Pm_-nWBHxYxVNopVhrd80Y4-2c
method: GET
uri: https://[::1]:35357/v3/projects?name=gnocchi
response:
body:
encoding: null
string: |-
{
"projects": [],
"links": {
"self": "https://[::1]:5000/v3/projects?name=gnocchi",
"next": null,
"previous": null
}
}
headers:
Connection:
- Keep-Alive
Content-Length:
- '134'
Content-Type:
- application/json
Date:
- Mon, 15 May 2017 07:49:53 GMT
Keep-Alive:
- timeout=3, max=96
Server:
- Apache/2.4.18 (Ubuntu)
Strict-Transport-Security:
- max-age=15768000
Vary:
- X-Auth-Token
X-Distribution:
- Ubuntu
x-openstack-request-id:
- req-fdeed726-18a4-4e73-bf8d-d24a5b56246e
status:
code: 200
message: OK
url: https://[::1]:35357/v3/projects?name=gnocchi
recorded_with: betamax/0.8.0

View File

@ -13,6 +13,8 @@
# License for the specific language governing permissions and limitations
# under the License.
from threading import Event
import mock
from oslo_config import cfg
from oslo_utils import timeutils
@ -20,7 +22,7 @@ import six
from ceilometermiddleware import swift
from ceilometermiddleware.tests import base as tests_base
from threading import Event
from keystoneauth1.fixture import keystoneauth_betamax as betamax
class FakeApp(object):
@ -431,3 +433,31 @@ class TestSwift(tests_base.TestCase):
with mock.patch('oslo_messaging.Notifier.info') as notify:
list(app(req.environ, self.start_response))
self.assertFalse(notify.called)
def test_ignore_projects_without_keystone(self):
app = swift.Swift(FakeApp(), {
'ignore_projects': 'cf0356aaac7c42bba5a744339a6169fa,'
'18157dd635bb413c9e27686fee93c583',
})
self.assertEqual(["cf0356aaac7c42bba5a744339a6169fa",
"18157dd635bb413c9e27686fee93c583"],
app.ignore_projects)
@mock.patch.object(swift._LOG, 'warning')
def test_ignore_projects_with_keystone(self, warning):
self.useFixture(betamax.BetamaxFixture(
cassette_name='list_projects',
cassette_library_dir='ceilometermiddleware/tests/data',
))
app = swift.Swift(FakeApp(), {
'auth_type': 'v2password',
'auth_url': 'https://[::1]:5000/v2.0',
'username': 'admin',
'tenant_name': 'admin',
'password': 'secret',
'ignore_projects': 'service,gnocchi',
})
self.assertEqual(["147cc0a9263c4964926f3ee7b6ba3685"],
app.ignore_projects)
warning.assert_called_once_with(
"fail to find project '%s' in keystone", "gnocchi")

View File

@ -8,3 +8,5 @@ oslo.utils>=3.5.0 # Apache-2.0
pbr>=1.6 # Apache-2.0
pycadf!=2.0.0,>=1.1.0 # Apache-2.0
six>=1.9.0 # MIT
keystoneauth1>=2.10.0 # Apache-2.0
python-keystoneclient>=2.0.0,!=2.1.0 # Apache-2.0

View File

@ -9,3 +9,4 @@ oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
oslotest>=1.10.0 # Apache-2.0
testrepository>=0.0.18 # Apache-2.0/BSD
mock>=1.2 # BSD
betamax>=0.7.0 # Apache-2.0