From f320089c152877db8cc3c16ed77dec7c12f9f152 Mon Sep 17 00:00:00 2001 From: Hongbin Lu Date: Sun, 6 Jan 2019 19:06:12 +0000 Subject: [PATCH] Support private registry - DB layer Add database table 'registry'. This table will store information about a registry, such as domain, username and password. Change-Id: If5532555e09d909d76025eafc99b5f6b2eeef19e Partial-Bug: #1702830 --- zun/common/exception.py | 8 + zun/db/api.py | 83 +++++++++ .../5ffc1cabe6b4_add_registry_table.py | 49 +++++ zun/db/sqlalchemy/api.py | 78 ++++++++ zun/db/sqlalchemy/models.py | 18 ++ zun/tests/unit/db/test_registry.py | 173 ++++++++++++++++++ zun/tests/unit/db/utils.py | 24 +++ 7 files changed, 433 insertions(+) create mode 100644 zun/db/sqlalchemy/alembic/versions/5ffc1cabe6b4_add_registry_table.py create mode 100644 zun/tests/unit/db/test_registry.py diff --git a/zun/common/exception.py b/zun/common/exception.py index d5c815268..ef9e66629 100644 --- a/zun/common/exception.py +++ b/zun/common/exception.py @@ -386,6 +386,10 @@ class NetworkNotFound(HTTPNotFound): message = _("Neutron network %(network)s could not be found.") +class RegistryNotFound(HTTPNotFound): + message = _("Registry %(registry)s could not be found.") + + class NetworkAlreadyExists(ResourceExists): message = _("A network with %(field)s %(value)s already exists.") @@ -472,6 +476,10 @@ class VolumeAlreadyExists(ResourceExists): message = _("A volume with %(field)s %(value)s already exists.") +class RegistryAlreadyExists(ResourceExists): + message = _("A registry with %(field)s %(value)s already exists.") + + class PortNotUsable(Invalid): message = _("Port %(port)s not usable for the container.") diff --git a/zun/db/api.py b/zun/db/api.py index ae5f36dfe..5e0d9a6df 100644 --- a/zun/db/api.py +++ b/zun/db/api.py @@ -1163,3 +1163,86 @@ def list_exec_instances(context, filters=None, limit=None, marker=None, @profiler.trace('db') def count_usage(context, project_id, flag): return _get_dbdriver_instance().count_usage(context, project_id, flag) + + +@profiler.trace("db") +def create_registry(context, values): + """Create a new registry. + + :param context: The security context + :param values: A dict containing several items used to identify + and track the registry, and several dicts which are + passed + into the Drivers when managing this registry. + :returns: A registry. + """ + return _get_dbdriver_instance().create_registry(context, values) + + +@profiler.trace("db") +def get_registry_by_uuid(context, registry_uuid): + """Return a registry. + + :param context: The security context + :param registry_uuid: The uuid of a registry. + :returns: A registry. + """ + return _get_dbdriver_instance().get_registry_by_uuid( + context, registry_uuid) + + +@profiler.trace("db") +def get_registry_by_name(context, registry_name): + """Return a registry. + + :param context: The security context + :param registry_name: The name of a registry. + :returns: A registry. + """ + return _get_dbdriver_instance().get_registry_by_name( + context, registry_name) + + +@profiler.trace("db") +def list_registries(context, filters=None, limit=None, marker=None, + sort_key=None, sort_dir=None): + """List matching registries. + + Return a list of the specified columns for all registries that match + the specified filters. + + :param context: The security context + :param filters: Filters to apply. Defaults to None. + :param limit: Maximum number of registries to return. + :param marker: the last item of the previous page; we return the next + result set. + :param sort_key: Attribute by which results should be sorted. + :param sort_dir: Direction in which results should be sorted. + (asc, desc) + :returns: A list of tuples of the specified columns. + """ + return _get_dbdriver_instance().list_registries( + context, filters, limit, marker, sort_key, sort_dir) + + +@profiler.trace("db") +def update_registry(context, uuid, values): + """Update properties of a registry. + + :param context: Request context + :param uuid: The id or uuid of a registry. + :param values: The properties to be updated + :returns: A registry. + """ + return _get_dbdriver_instance().update_registry( + context, uuid, values) + + +@profiler.trace("db") +def destroy_registry(context, registry_uuid): + """Destroy a registry. + + :param context: Request context + :param registry_uuid: The uuid of a registry. + """ + return _get_dbdriver_instance().destroy_registry(context, registry_uuid) diff --git a/zun/db/sqlalchemy/alembic/versions/5ffc1cabe6b4_add_registry_table.py b/zun/db/sqlalchemy/alembic/versions/5ffc1cabe6b4_add_registry_table.py new file mode 100644 index 000000000..ade8e6fec --- /dev/null +++ b/zun/db/sqlalchemy/alembic/versions/5ffc1cabe6b4_add_registry_table.py @@ -0,0 +1,49 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""add registry table + +Revision ID: 5ffc1cabe6b4 +Revises: 21fa080c818a +Create Date: 2019-01-04 02:22:45.889795 + +""" + +# revision identifiers, used by Alembic. +revision = '5ffc1cabe6b4' +down_revision = '21fa080c818a' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table( + 'registry', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.String(length=255), nullable=True), + sa.Column('user_id', sa.String(length=255), nullable=True), + sa.Column('uuid', sa.String(length=36), nullable=True), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('domain', sa.String(length=255), nullable=False), + sa.Column('username', sa.String(length=255), nullable=True), + sa.Column('password', sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('uuid', name='uniq_registry0uuid'), + mysql_charset='utf8', + mysql_engine='InnoDB' + ) diff --git a/zun/db/sqlalchemy/api.py b/zun/db/sqlalchemy/api.py index 9e079b267..11438dba6 100644 --- a/zun/db/sqlalchemy/api.py +++ b/zun/db/sqlalchemy/api.py @@ -1353,3 +1353,81 @@ class Connection(object): filter_by(project_id=project_id) return project_query.first() + + def _add_registries_filters(self, query, filters): + filter_names = ['name', 'domain', 'username', 'project_id', 'user_id'] + return self._add_filters(query, models.Registry, filters=filters, + filter_names=filter_names) + + def list_registries(self, context, filters=None, limit=None, + marker=None, sort_key=None, sort_dir=None): + query = model_query(models.Registry) + query = self._add_project_filters(context, query) + query = self._add_registries_filters(query, filters) + return _paginate_query(models.Registry, limit, marker, + sort_key, sort_dir, query) + + def create_registry(self, context, values): + # ensure defaults are present for new registries + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + registry = models.Registry() + registry.update(values) + try: + registry.save() + except db_exc.DBDuplicateEntry: + raise exception.RegistryAlreadyExists( + field='UUID', value=values['uuid']) + return registry + + def update_registry(self, context, registry_uuid, values): + # NOTE(dtantsur): this can lead to very strange errors + if 'uuid' in values: + msg = _("Cannot overwrite UUID for an existing registry.") + raise exception.InvalidParameterValue(err=msg) + return self._do_update_registry(registry_uuid, values) + + def _do_update_registry(self, registry_uuid, values): + session = get_session() + with session.begin(): + query = model_query(models.Registry, session=session) + query = add_identity_filter(query, registry_uuid) + try: + ref = query.with_lockmode('update').one() + except NoResultFound: + raise exception.RegistryNotFound(registry=registry_uuid) + + ref.update(values) + return ref + + def get_registry_by_uuid(self, context, registry_uuid): + query = model_query(models.Registry) + query = self._add_project_filters(context, query) + query = query.filter_by(uuid=registry_uuid) + try: + return query.one() + except NoResultFound: + raise exception.RegistryNotFound(registry=registry_uuid) + + def get_registry_by_name(self, context, registry_name): + query = model_query(models.Registry) + query = self._add_project_filters(context, query) + query = query.filter_by(name=registry_name) + try: + return query.one() + except NoResultFound: + raise exception.RegistryNotFound(registry=registry_name) + except MultipleResultsFound: + raise exception.Conflict('Multiple registries exist with same ' + 'name. Please use the registry uuid ' + 'instead.') + + def destroy_registry(self, context, registry_uuid): + session = get_session() + with session.begin(): + query = model_query(models.Registry, session=session) + query = add_identity_filter(query, registry_uuid) + count = query.delete() + if count != 1: + raise exception.RegistryNotFound(registry=registry_uuid) diff --git a/zun/db/sqlalchemy/models.py b/zun/db/sqlalchemy/models.py index 842858779..53f7f3fde 100644 --- a/zun/db/sqlalchemy/models.py +++ b/zun/db/sqlalchemy/models.py @@ -590,3 +590,21 @@ class Network(Base): project_id = Column(String(255)) user_id = Column(String(255)) uuid = Column(String(36)) + + +class Registry(Base): + """Represents a registry. """ + + __tablename__ = 'registry' + __table_args__ = ( + schema.UniqueConstraint('uuid', name='uniq_registry0uuid'), + table_args() + ) + id = Column(Integer, primary_key=True) + project_id = Column(String(255)) + user_id = Column(String(255)) + uuid = Column(String(36)) + name = Column(String(255)) + domain = Column(String(255)) + username = Column(String(255)) + password = Column(String(255)) diff --git a/zun/tests/unit/db/test_registry.py b/zun/tests/unit/db/test_registry.py new file mode 100644 index 000000000..d1961379d --- /dev/null +++ b/zun/tests/unit/db/test_registry.py @@ -0,0 +1,173 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_utils import uuidutils +import six + +from zun.common import exception +import zun.conf +from zun.db import api as dbapi +from zun.tests.unit.db import base +from zun.tests.unit.db import utils + + +CONF = zun.conf.CONF + + +class DbRegistryTestCase(base.DbTestCase): + + def setUp(self): + super(DbRegistryTestCase, self).setUp() + + def test_create_registry(self): + utils.create_test_registry(context=self.context) + + def test_create_registry_already_exists(self): + utils.create_test_registry(context=self.context, + uuid='123') + with self.assertRaisesRegex(exception.RegistryAlreadyExists, + 'A registry with UUID 123.*'): + utils.create_test_registry(context=self.context, + uuid='123') + + def test_get_registry_by_uuid(self): + registry = utils.create_test_registry(context=self.context) + res = dbapi.get_registry_by_uuid(self.context, + registry.uuid) + self.assertEqual(registry.id, res.id) + self.assertEqual(registry.uuid, res.uuid) + + def test_get_registry_by_name(self): + registry = utils.create_test_registry(context=self.context) + res = dbapi.get_registry_by_name( + self.context, registry.name) + self.assertEqual(registry.id, res.id) + self.assertEqual(registry.uuid, res.uuid) + + def test_get_registry_that_does_not_exist(self): + self.assertRaises(exception.RegistryNotFound, + dbapi.get_registry_by_uuid, + self.context, + uuidutils.generate_uuid()) + + def test_list_registries(self): + uuids = [] + for i in range(1, 6): + registry = utils.create_test_registry( + uuid=uuidutils.generate_uuid(), + context=self.context, + name='registry' + str(i)) + uuids.append(six.text_type(registry['uuid'])) + res = dbapi.list_registries(self.context) + res_uuids = [r.uuid for r in res] + self.assertEqual(sorted(uuids), sorted(res_uuids)) + + def test_list_registries_sorted(self): + uuids = [] + for i in range(5): + registry = utils.create_test_registry( + uuid=uuidutils.generate_uuid(), + context=self.context, + name='registry' + str(i)) + uuids.append(six.text_type(registry.uuid)) + res = dbapi.list_registries(self.context, sort_key='uuid') + res_uuids = [r.uuid for r in res] + self.assertEqual(sorted(uuids), res_uuids) + + self.assertRaises(exception.InvalidParameterValue, + dbapi.list_registries, + self.context, + sort_key='foo') + + def test_list_registries_with_filters(self): + registry1 = utils.create_test_registry( + name='registry-one', + uuid=uuidutils.generate_uuid(), + context=self.context) + registry2 = utils.create_test_registry( + name='registry-two', + uuid=uuidutils.generate_uuid(), + context=self.context) + + res = dbapi.list_registries( + self.context, filters={'name': 'registry-one'}) + self.assertEqual([registry1.id], [r.id for r in res]) + + res = dbapi.list_registries( + self.context, filters={'name': 'registry-two'}) + self.assertEqual([registry2.id], [r.id for r in res]) + + res = dbapi.list_registries( + self.context, filters={'name': 'bad-registry'}) + self.assertEqual([], [r.id for r in res]) + + res = dbapi.list_registries( + self.context, + filters={'name': registry1.name}) + self.assertEqual([registry1.id], [r.id for r in res]) + + def test_list_registries_with_list_filters(self): + registry1 = utils.create_test_registry( + name='registry-one', + uuid=uuidutils.generate_uuid(), + context=self.context) + registry2 = utils.create_test_registry( + name='registry-two', + uuid=uuidutils.generate_uuid(), + context=self.context) + + res = dbapi.list_registries( + self.context, filters={'name': ['registry-one', 'registry-two']}) + uuids = sorted([registry1.uuid, registry2.uuid]) + self.assertEqual(uuids, sorted([r.uuid for r in res])) + + def test_destroy_registry(self): + registry = utils.create_test_registry(context=self.context) + dbapi.destroy_registry(self.context, registry.id) + self.assertRaises(exception.RegistryNotFound, + dbapi.get_registry_by_uuid, + self.context, registry.uuid) + + def test_destroy_registry_by_uuid(self): + registry = utils.create_test_registry(context=self.context) + dbapi.destroy_registry(self.context, registry.uuid) + self.assertRaises(exception.RegistryNotFound, + dbapi.get_registry_by_uuid, + self.context, registry.uuid) + + def test_destroy_registry_that_does_not_exist(self): + self.assertRaises(exception.RegistryNotFound, + dbapi.destroy_registry, self.context, + uuidutils.generate_uuid()) + + def test_update_registry(self): + registry = utils.create_test_registry(context=self.context) + old_name = registry.name + new_name = 'new-name' + self.assertNotEqual(old_name, new_name) + + res = dbapi.update_registry(self.context, registry.id, + {'name': new_name}) + self.assertEqual(new_name, res.name) + + def test_update_registry_not_found(self): + registry_uuid = uuidutils.generate_uuid() + new_name = 'new-name' + self.assertRaises(exception.RegistryNotFound, + dbapi.update_registry, self.context, + registry_uuid, {'name': new_name}) + + def test_update_registry_uuid(self): + registry = utils.create_test_registry(context=self.context) + self.assertRaises(exception.InvalidParameterValue, + dbapi.update_registry, self.context, + registry.id, {'uuid': ''}) diff --git a/zun/tests/unit/db/utils.py b/zun/tests/unit/db/utils.py index d3b6aad9e..30c727a10 100644 --- a/zun/tests/unit/db/utils.py +++ b/zun/tests/unit/db/utils.py @@ -665,3 +665,27 @@ def create_test_network(**kwargs): del network['id'] dbapi = _get_dbapi() return dbapi.create_network(kwargs['context'], network) + + +def get_test_registry(**kwargs): + return { + 'id': kwargs.get('id', 42), + 'name': kwargs.get('name', 'fake_name'), + 'uuid': kwargs.get('uuid', '0b5cdde8-237a-4917-9556-003e5c588c4f'), + 'project_id': kwargs.get('project_id', 'fake_project'), + 'user_id': kwargs.get('user_id', 'fake_user'), + 'created_at': kwargs.get('created_at'), + 'updated_at': kwargs.get('updated_at'), + 'domain': kwargs.get('domain', 'test.io'), + 'username': kwargs.get('username', 'fake_username'), + 'password': kwargs.get('password', 'fake_password'), + } + + +def create_test_registry(**kwargs): + registry = get_test_registry(**kwargs) + # Let DB generate ID if it isn't specified explicitly + if 'id' not in kwargs: + del registry['id'] + dbapi = _get_dbapi() + return dbapi.create_registry(kwargs['context'], registry)