From 555d0c885692200b6ab3dbb3adf780784e389183 Mon Sep 17 00:00:00 2001 From: Jim Baker Date: Wed, 29 Mar 2017 11:25:22 -0600 Subject: [PATCH] Updates Alembic migration to better match SQLAlchemy models The original Alembic migration script was autogenerated, then subsequently modified by hand. Between these two aspects, some correspondence to the corresponding SQLAlchemy models was lost: * Columns were specified independently of corresponding foreign key and primary key constraints; besides making it more difficult to follow the code between models and the migration script, there was a loss of integrity constraints: a variable association must exist for every entity that mixes in VariableMixin as a base was not being enforced at the database level with an appropriate not-null; this causes problems for use of variables for any such object that is not constructed by models code, most notably via direct usage of the database. (See specifically https://bugs.launchpad.net/craton/+bug/1668251, fixed with this change.) * The column created_at would always be set by model-using code; but this was not enforced at the database level by ensuring it was not null. * Children should be deleted if the parent is deleted (we do not support "soft delete" in Craton). In addition, all constraints are now named (the original intent of this change, so as to ensure the feasibility of future migrations), and they are also named in a consistent fashion. Note that this change does not support many-to-many deletions (as seen in https://bugs.launchpad.net/craton/+bug/1668308), but SQLAlchemy does provide support for this at the model level. A future change can address this without requiring Alembic support. There's no direct testing of this change, but it is implicitly, and rather strongly, tested by the overall functional tests that are used; and to a lesser extent by some of the unit tests. This is in part because this change has strengthened constraints, not weakened them. Change-Id: I1f84c29610127de12c292a210fd003ae07bd6462 Closes-bug: 1665066 Closes-bug: 1668251 --- .../ffdc1a500db1_craton_inventory_init.py | 449 ++++++++++-------- craton/db/sqlalchemy/models.py | 39 +- craton/tests/functional/__init__.py | 69 +-- tools/docker_run.sh | 1 - 4 files changed, 303 insertions(+), 255 deletions(-) diff --git a/craton/db/sqlalchemy/alembic/versions/ffdc1a500db1_craton_inventory_init.py b/craton/db/sqlalchemy/alembic/versions/ffdc1a500db1_craton_inventory_init.py index 245bd2a..c0dc28c 100644 --- a/craton/db/sqlalchemy/alembic/versions/ffdc1a500db1_craton_inventory_init.py +++ b/craton/db/sqlalchemy/alembic/versions/ffdc1a500db1_craton_inventory_init.py @@ -20,269 +20,340 @@ import sqlalchemy_utils def upgrade(): op.create_table( 'variable_association', - sa.Column('created_at', sa.DateTime, nullable=True), + sa.Column('created_at', sa.DateTime, nullable=False), sa.Column('updated_at', sa.DateTime, nullable=True), - sa.Column('id', sa.Integer, nullable=False), + sa.Column('id', sa.Integer, primary_key=True), sa.Column('discriminator', sa.String(length=50), nullable=False), - sa.PrimaryKeyConstraint('id') ) + op.create_table( 'variables', - sa.Column('created_at', sa.DateTime, nullable=True), + sa.Column('created_at', sa.DateTime, nullable=False), sa.Column('updated_at', sa.DateTime, nullable=True), - sa.Column('association_id', sa.Integer), - sa.Column('key_', sa.String(length=255), nullable=False), - sa.Column('value_', sa.JSON, - nullable=True), - sa.PrimaryKeyConstraint('association_id', 'key_'), - sa.ForeignKeyConstraint( - ['association_id'], ['variable_association.id'], - 'fk_variables_variable_association') - ) - op.create_table( - 'access_secrets', - sa.Column('created_at', sa.DateTime, nullable=True), - sa.Column('updated_at', sa.DateTime, nullable=True), - sa.Column('id', sa.Integer, nullable=False), - sa.Column('cert', sa.Text(), nullable=True), - sa.PrimaryKeyConstraint('id') + sa.Column( + 'association_id', sa.Integer, + sa.ForeignKey( + 'variable_association.id', + name='fk_variables__variables_association', + ondelete='cascade'), + primary_key=True), + sa.Column('key_', sa.String(length=255), primary_key=True), + sa.Column('value_', sa.JSON, nullable=True), ) + op.create_table( 'projects', - sa.Column('created_at', sa.DateTime, nullable=True), + sa.Column('created_at', sa.DateTime, nullable=False), sa.Column('updated_at', sa.DateTime, nullable=True), - sa.Column('id', sqlalchemy_utils.types.UUIDType(binary=False), - nullable=False), - sa.Column('variable_association_id', sa.Integer), + sa.Column( + 'id', sqlalchemy_utils.types.UUIDType(binary=False), + primary_key=True), + sa.Column( + 'variable_association_id', sa.Integer, + sa.ForeignKey( + 'variable_association.id', + name='fk_projects__variable_association'), + nullable=False), sa.Column('name', sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.ForeignKeyConstraint( - ['variable_association_id'], ['variable_association.id'], - 'fk_projects_variable_association') ) - op.create_table( - 'clouds', - sa.Column('created_at', sa.DateTime, nullable=True), - sa.Column('updated_at', sa.DateTime, nullable=True), - sa.Column('id', sa.Integer, nullable=False), - sa.Column('project_id', sqlalchemy_utils.types.UUIDType(binary=False), - nullable=False), - sa.Column('variable_association_id', sa.Integer), - sa.Column('name', sa.String(length=255), nullable=True), - sa.Column('note', sa.Text, nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint( - 'project_id', 'name', - name='uq_cloud0projectid0name'), - sa.ForeignKeyConstraint(['project_id'], ['projects.id']), - sa.ForeignKeyConstraint( - ['variable_association_id'], ['variable_association.id'], - 'fk_clouds_variable_association') - ) - op.create_index(op.f('ix_clouds_project_id'), - 'clouds', ['project_id'], unique=False) - op.create_table( - 'regions', - sa.Column('created_at', sa.DateTime, nullable=True), - sa.Column('updated_at', sa.DateTime, nullable=True), - sa.Column('id', sa.Integer, nullable=False), - sa.Column('project_id', sqlalchemy_utils.types.UUIDType(binary=False), - nullable=False), - sa.Column('cloud_id', sa.Integer, nullable=False), - sa.Column('variable_association_id', sa.Integer), - sa.Column('name', sa.String(length=255), nullable=True), - sa.Column('note', sa.Text, nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint( - 'cloud_id', 'name', - name='uq_region0cloudid0name'), - sa.ForeignKeyConstraint(['project_id'], ['projects.id']), - sa.ForeignKeyConstraint(['cloud_id'], ['clouds.id']), - sa.ForeignKeyConstraint( - ['variable_association_id'], ['variable_association.id'], - 'fk_regions_variable_association') - ) - op.create_index(op.f('ix_regions_project_id'), - 'regions', ['project_id'], unique=False) - op.create_index(op.f('ix_regions_cloud_id'), - 'regions', ['cloud_id'], unique=False) + op.create_table( 'users', - sa.Column('created_at', sa.DateTime, nullable=True), + sa.Column('created_at', sa.DateTime, nullable=False), sa.Column('updated_at', sa.DateTime, nullable=True), - sa.Column('id', sa.Integer, nullable=False), - sa.Column('project_id', sqlalchemy_utils.types.UUIDType(binary=False), - nullable=False), - sa.Column('variable_association_id', sa.Integer), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column( + 'project_id', sqlalchemy_utils.types.UUIDType(binary=False), + sa.ForeignKey( + 'projects.id', name='fk_users__projects', ondelete='cascade'), + nullable=False), + sa.Column( + 'variable_association_id', sa.Integer, + sa.ForeignKey( + 'variable_association.id', + name='fk_users__variable_association'), + nullable=False), sa.Column('username', sa.String(length=255), nullable=True), sa.Column('api_key', sa.String(length=36), nullable=True), sa.Column('is_root', sa.Boolean, nullable=True), sa.Column('is_admin', sa.Boolean, nullable=True), - sa.Column('roles', sa.JSON, - nullable=True), - sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint( 'username', 'project_id', - name='uq_user0username0project'), - sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ), - sa.ForeignKeyConstraint( - ['variable_association_id'], ['variable_association.id'], - 'fk_users_variable_association') + name='uq_users_username_project_id'), ) - op.create_index(op.f('ix_users_project_id'), 'users', ['project_id'], - unique=False) + op.create_index( + op.f('ix_users_project_id'), 'users', ['project_id'], unique=False) + op.create_table( - 'cells', - sa.Column('created_at', sa.DateTime, nullable=True), + 'clouds', + sa.Column('created_at', sa.DateTime, nullable=False), sa.Column('updated_at', sa.DateTime, nullable=True), - sa.Column('id', sa.Integer, nullable=False), - sa.Column('project_id', sqlalchemy_utils.types.UUIDType(binary=False), - nullable=False), - sa.Column('cloud_id', sa.Integer, nullable=False), - sa.Column('region_id', sa.Integer, nullable=False), - sa.Column('variable_association_id', sa.Integer), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column( + 'project_id', sqlalchemy_utils.types.UUIDType(binary=False), + sa.ForeignKey( + 'projects.id', name='fk_clouds__projects', ondelete='cascade'), + nullable=False), + sa.Column( + 'variable_association_id', sa.Integer, + sa.ForeignKey( + 'variable_association.id', + name='fk_clouds__variable_association'), + nullable=False), sa.Column('name', sa.String(length=255), nullable=True), sa.Column('note', sa.Text, nullable=True), - sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint( - 'region_id', 'name', name='uq_cell0regionid0name'), - sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ), - sa.ForeignKeyConstraint(['cloud_id'], ['clouds.id'], ), - sa.ForeignKeyConstraint(['region_id'], ['regions.id'], ), - sa.ForeignKeyConstraint( - ['variable_association_id'], ['variable_association.id'], - 'fk_cells_variable_association') + 'project_id', 'name', + name='uq_clouds__project_id__name'), ) op.create_index( - op.f('ix_cells_project_id'), 'cells', ['project_id'], - unique=False) + op.f('ix_clouds_project_id'), 'clouds', ['project_id'], unique=False) + + op.create_table( + 'regions', + sa.Column('created_at', sa.DateTime, nullable=False), + sa.Column('updated_at', sa.DateTime, nullable=True), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column( + 'project_id', sqlalchemy_utils.types.UUIDType(binary=False), + sa.ForeignKey( + 'projects.id', + name='fk_projects__regions', ondelete='cascade')), + sa.Column( + 'cloud_id', sa.Integer, + sa.ForeignKey( + 'clouds.id', name='fk_regions__clouds', ondelete='cascade'), + nullable=False), + sa.Column( + 'variable_association_id', sa.Integer, + sa.ForeignKey( + 'variable_association.id', + name='fk_regions__variable_association'), + nullable=False), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('note', sa.Text, nullable=True), + sa.UniqueConstraint( + 'cloud_id', 'name', name='uq_regions__cloud_id__name'), + ) + op.create_index( - op.f('ix_cells_cloud_id'), 'cells', ['cloud_id'], - unique=False) + op.f('ix_regions_project_id'), 'regions', ['project_id'], unique=False) op.create_index( - op.f('ix_cells_region_id'), 'cells', ['region_id'], - unique=False) + op.f('ix_regions_cloud_id'), 'regions', ['cloud_id'], unique=False) + + op.create_table( + 'cells', + sa.Column('created_at', sa.DateTime, nullable=False), + sa.Column('updated_at', sa.DateTime, nullable=True), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column( + 'project_id', sqlalchemy_utils.types.UUIDType(binary=False), + sa.ForeignKey( + 'projects.id', name='fk_cells__projects', ondelete='cascade'), + nullable=False), + sa.Column( + 'cloud_id', sa.Integer, + sa.ForeignKey( + 'clouds.id', name='fk_cells__clouds', ondelete='cascade'), + nullable=False), + sa.Column( + 'region_id', sa.Integer, + sa.ForeignKey( + 'regions.id', name='fk_cells__regions', ondelete='cascade'), + nullable=False), + sa.Column( + 'variable_association_id', sa.Integer, + sa.ForeignKey( + 'variable_association.id', + name='fk_cells__variable_association'), + nullable=False), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('note', sa.Text, nullable=True), + sa.UniqueConstraint( + 'region_id', 'name', name='uq_cells__region_id__name'), + ) + op.create_index( + op.f('ix_cells_project_id'), 'cells', ['project_id'], unique=False) + op.create_index( + op.f('ix_cells_cloud_id'), 'cells', ['cloud_id'], unique=False) + op.create_index( + op.f('ix_cells_region_id'), 'cells', ['region_id'], unique=False) + op.create_table( 'networks', - sa.Column('created_at', sa.DateTime, nullable=True), + sa.Column('created_at', sa.DateTime, nullable=False), sa.Column('updated_at', sa.DateTime, nullable=True), - sa.Column('id', sa.Integer, nullable=False), - sa.Column('project_id', sqlalchemy_utils.types.UUIDType(binary=False), - nullable=False), - sa.Column('cloud_id', sa.Integer, nullable=False), - sa.Column('region_id', sa.Integer, nullable=False), - sa.Column('cell_id', sa.Integer, nullable=True), - sa.Column('variable_association_id', sa.Integer), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column( + 'project_id', sqlalchemy_utils.types.UUIDType(binary=False), + sa.ForeignKey( + 'projects.id', + name='fk_networks__projects', ondelete='cascade'), + nullable=False), + sa.Column( + 'cloud_id', sa.Integer, + sa.ForeignKey( + 'clouds.id', name='fk_networks__clouds', ondelete='cascade'), + nullable=False), + sa.Column( + 'region_id', sa.Integer, + sa.ForeignKey( + 'regions.id', name='fk_networks__regions', ondelete='cascade'), + nullable=False), + sa.Column( + 'cell_id', sa.Integer, + sa.ForeignKey( + 'cells.id', name='fk_networks__cells', ondelete='cascade'), + nullable=True), + sa.Column( + 'variable_association_id', sa.Integer, + sa.ForeignKey( + 'variable_association.id', + name='fk_networks__variable_association'), + nullable=False), sa.Column('name', sa.String(length=255), nullable=True), sa.Column('cidr', sa.String(length=255), nullable=True), sa.Column('gateway', sa.String(length=255), nullable=True), sa.Column('netmask', sa.String(length=255), nullable=True), sa.Column('ip_block_type', sa.String(length=255), nullable=True), sa.Column('nss', sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint("name", "project_id", "region_id", - name="uq_name0projectid0regionid"), - sa.ForeignKeyConstraint(['project_id'], ['projects.id']), - sa.ForeignKeyConstraint(['cloud_id'], ['clouds.id']), - sa.ForeignKeyConstraint(['region_id'], ['regions.id']), - sa.ForeignKeyConstraint(['cell_id'], ['cells.id']), - sa.ForeignKeyConstraint( - ['variable_association_id'], ['variable_association.id'], - 'fk_networks_variable_association') + sa.UniqueConstraint( + 'name', 'project_id', 'region_id', + name='uq_networks__name__project_id__region_id'), ) op.create_index( - op.f('ix_networks_cell_id'), 'networks', ['cell_id'], - unique=False) + op.f('ix_networks_cell_id'), 'networks', ['cell_id'], unique=False) op.create_index( op.f('ix_networks_project_id'), 'networks', ['project_id'], unique=False) op.create_index( op.f('ix_networks_region_id'), 'networks', ['region_id'], unique=False) + op.create_table( 'devices', - sa.Column('created_at', sa.DateTime, nullable=True), + sa.Column('created_at', sa.DateTime, nullable=False), sa.Column('updated_at', sa.DateTime, nullable=True), - sa.Column('id', sa.Integer, nullable=False), - sa.Column('type', sa.String(length=50), nullable=True), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column( + 'project_id', sqlalchemy_utils.types.UUIDType(binary=False), + sa.ForeignKey( + 'projects.id', + name='fk_devices__projects', ondelete='cascade'), + nullable=False), + sa.Column( + 'cloud_id', sa.Integer, + sa.ForeignKey( + 'clouds.id', name='fk_devices__clouds', ondelete='cascade'), + nullable=False), + sa.Column( + 'region_id', sa.Integer, + sa.ForeignKey( + 'regions.id', name='fk_devices__regions', ondelete='cascade'), + nullable=False), + sa.Column( + 'cell_id', sa.Integer, + sa.ForeignKey( + 'cells.id', name='fk_devices__cells', ondelete='cascade'), + nullable=True), + sa.Column( + 'parent_id', sa.Integer, + sa.ForeignKey('devices.id', name='fk_devices__devices'), + nullable=True), + sa.Column( + 'variable_association_id', sa.Integer, + sa.ForeignKey( + 'variable_association.id', + name='fk_devices__variable_association'), + nullable=False), + sa.Column('type', sa.String(length=50), nullable=False), sa.Column('device_type', sa.String(length=255), nullable=False), sa.Column('name', sa.String(length=255), nullable=False), - sa.Column('project_id', sqlalchemy_utils.types.UUIDType(binary=False), - nullable=False), - sa.Column('cloud_id', sa.Integer, nullable=False), - sa.Column('region_id', sa.Integer, nullable=False), - sa.Column('cell_id', sa.Integer, nullable=True), - sa.Column('parent_id', sa.Integer, nullable=True), - sa.Column('access_secret_id', sa.Integer, nullable=True), - sa.Column('variable_association_id', sa.Integer), - sa.Column('ip_address', - sqlalchemy_utils.types.IPAddressType(length=64), - nullable=False), + sa.Column( + 'ip_address', + sqlalchemy_utils.types.IPAddressType(length=64), + nullable=False), sa.Column('active', sa.Boolean(), nullable=True), sa.Column('note', sa.Text(), nullable=True), - sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('region_id', 'name', name='uq_device0regionid0name'), - sa.ForeignKeyConstraint(['project_id'], ['projects.id']), - sa.ForeignKeyConstraint(['cloud_id'], ['clouds.id']), - sa.ForeignKeyConstraint(['region_id'], ['regions.id']), - sa.ForeignKeyConstraint(['cell_id'], ['cells.id']), - sa.ForeignKeyConstraint(['access_secret_id'], ['access_secrets.id']), - sa.ForeignKeyConstraint(['parent_id'], ['devices.id']), - sa.ForeignKeyConstraint( - ['variable_association_id'], ['variable_association.id'], - 'fk_devices_variable_association') ) op.create_index( - op.f('ix_devices_cell_id'), 'devices', ['cell_id'], - unique=False) + op.f('ix_devices_cell_id'), 'devices', ['cell_id'], unique=False) op.create_index( - op.f('ix_devices_project_id'), 'devices', ['project_id'], - unique=False) + op.f('ix_devices_project_id'), 'devices', ['project_id'], unique=False) op.create_index( - op.f('ix_devices_region_id'), 'devices', ['region_id'], - unique=False) + op.f('ix_devices_region_id'), 'devices', ['region_id'], unique=False) op.create_index( - op.f('ix_devices_cloud_id'), 'devices', ['cloud_id'], - unique=False) + op.f('ix_devices_cloud_id'), 'devices', ['cloud_id'], unique=False) + op.create_table( 'hosts', sa.Column( 'id', sa.Integer, sa.ForeignKey( - 'devices.id', name='fk_hosts_devices', ondelete='cascade'), + 'devices.id', name='fk_hosts__devices', ondelete='cascade'), primary_key=True) ) + op.create_table( 'network_devices', sa.Column( 'id', sa.Integer, sa.ForeignKey( 'devices.id', - name='fk_network_devices_devices', ondelete='cascade'), + name='fk_network_devices__devices', ondelete='cascade'), primary_key=True), sa.Column('model_name', sa.String(length=255), nullable=True), sa.Column('os_version', sa.String(length=255), nullable=True), sa.Column('vlans', sa.JSON, nullable=True) ) + op.create_table( 'labels', - sa.Column('created_at', sa.DateTime, nullable=True), + sa.Column('created_at', sa.DateTime, nullable=False), sa.Column('updated_at', sa.DateTime, nullable=True), - sa.Column('device_id', sa.Integer, nullable=False), - sa.Column('label', sa.String(length=255), nullable=False), - sa.PrimaryKeyConstraint('device_id', 'label'), - sa.ForeignKeyConstraint( - ['device_id'], ['devices.id'], - 'fk_labels_devices') + sa.Column( + 'device_id', sa.Integer, + sa.ForeignKey( + 'devices.id', + name='fk_labels__devices', ondelete='cascade'), + primary_key=True), + sa.Column('label', sa.String(length=255), primary_key=True), ) + op.create_index( + op.f('ix_devices_labels'), 'labels', ['label', 'device_id']) + op.create_table( 'network_interfaces', - sa.Column('created_at', sa.DateTime, nullable=True), + sa.Column('created_at', sa.DateTime, nullable=False), sa.Column('updated_at', sa.DateTime, nullable=True), - sa.Column('id', sa.Integer, nullable=False), - sa.Column('variable_association_id', sa.Integer), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column( + 'project_id', sqlalchemy_utils.types.UUIDType(binary=False), + sa.ForeignKey( + 'projects.id', + name='fk_network_interfaces__projects', ondelete='cascade'), + nullable=False), + sa.Column( + 'device_id', sa.Integer, + sa.ForeignKey( + 'devices.id', + name='fk_network_interfaces__devices', ondelete='cascade'), + nullable=False), + sa.Column( + 'network_id', sa.Integer, + sa.ForeignKey( + 'networks.id', + name='fk_network_interfaces__networks'), + nullable=True), + sa.Column( + 'variable_association_id', sa.Integer, + sa.ForeignKey( + 'variable_association.id', + name='fk_network_interfaces__variable_association'), + nullable=False), sa.Column('name', sa.String(length=255), nullable=True), sa.Column('interface_type', sa.String(length=255), nullable=True), sa.Column('vlan_id', sa.Integer, nullable=True), @@ -294,30 +365,19 @@ def upgrade(): sa.Column('cdp', sa.String(length=255), nullable=True), sa.Column('security', sa.String(length=255), nullable=True), sa.Column( - 'device_id', sa.Integer, - sa.ForeignKey( - 'devices.id', - name='fk_network_interfaces_devices', ondelete='cascade'), + 'ip_address', + sqlalchemy_utils.types.IPAddressType(length=64), nullable=False), - sa.Column('network_id', sa.Integer, nullable=True), - sa.Column('project_id', sqlalchemy_utils.types.UUIDType(binary=False), - nullable=False), - sa.Column('ip_address', - sqlalchemy_utils.types.IPAddressType(length=64), - nullable=False), - sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint( - 'device_id', 'name', name='uq_netinter0deviceid0name'), - sa.ForeignKeyConstraint(['project_id'], ['projects.id']), - sa.ForeignKeyConstraint(['network_id'], ['networks.id'], ), - sa.ForeignKeyConstraint( - ['variable_association_id'], ['variable_association.id'], - 'fk_network_interfaces_variable_association') + 'device_id', 'name', + name='uq_network_interfaces__device_id__name'), ) def downgrade(): op.drop_table('network_interfaces') + op.drop_index(op.f('ix_devices_labels'), table_name='labels') + op.drop_table('labels') op.drop_table('network_devices') op.drop_table('hosts') op.drop_index(op.f('ix_networks_region_id'), table_name='networks') @@ -335,14 +395,13 @@ def downgrade(): op.drop_index(op.f('ix_cells_project_id'), table_name='cells') op.drop_table('cells') op.drop_index(op.f('ix_users_project_id'), table_name='users') - op.drop_table('users') op.drop_index(op.f('ix_regions_project_id'), table_name='regions') op.drop_index(op.f('ix_regions_cloud_id'), table_name='regions') op.drop_table('regions') op.drop_index(op.f('ix_clouds_project_id'), table_name='clouds') op.drop_table('clouds') + op.drop_table('users') op.drop_table('projects') - op.drop_table('labels') - op.drop_table('access_secrets') + op.drop_index(op.f('ix_variable_keys'), table_name='variables') op.drop_table('variables') op.drop_table('variable_association') diff --git a/craton/db/sqlalchemy/models.py b/craton/db/sqlalchemy/models.py index c68bc5f..38adbc4 100644 --- a/craton/db/sqlalchemy/models.py +++ b/craton/db/sqlalchemy/models.py @@ -244,6 +244,7 @@ class Project(Base, VariableMixin): devices = relationship('Device', back_populates='project') users = relationship('User', back_populates='project') networks = relationship('Network', back_populates='project') + interfaces = relationship('NetworkInterface', back_populates='project') class User(Base, VariableMixin): @@ -258,11 +259,11 @@ class User(Base, VariableMixin): nullable=False) username = Column(String(255)) api_key = Column(String(36)) - # root = craton admin that can create other pojects/usrs + # root = craton admin that can create other projects/users is_root = Column(Boolean, default=False) # admin = project context admin is_admin = Column(Boolean, default=False) - roles = Column(JSON) + _repr_columns = [id, username] project = relationship('Project', back_populates='users') @@ -371,7 +372,6 @@ class Device(Base, VariableMixin): project_id = Column( UUIDType(binary=False), ForeignKey('projects.id'), index=True, nullable=False) - access_secret_id = Column(Integer, ForeignKey('access_secrets.id')) parent_id = Column(Integer, ForeignKey('devices.id')) ip_address = Column(IPAddressType, nullable=False) device_type = Column(String(255), nullable=False) @@ -389,7 +389,6 @@ class Device(Base, VariableMixin): 'Label', back_populates='device', collection_class=set, cascade='all, delete-orphan', lazy='joined') labels = association_proxy('related_labels', 'label') - access_secret = relationship('AccessSecret', back_populates='devices') interfaces = relationship('NetworkInterface', back_populates='device') children = relationship( 'Device', backref=backref('parent', remote_side=[id])) @@ -462,7 +461,7 @@ class Host(Device): } -class NetworkInterface(Base): +class NetworkInterface(Base, VariableMixin): __tablename__ = 'network_interfaces' __table_args__ = ( UniqueConstraint("device_id", "name", @@ -485,10 +484,12 @@ class NetworkInterface(Base): device_id = Column(Integer, ForeignKey('devices.id')) network_id = Column(Integer, ForeignKey('networks.id'), nullable=True) - network = relationship('Network', back_populates="devices", - cascade='all', lazy='joined') - device = relationship('Device', back_populates="interfaces", - cascade='all', lazy='joined') + project = relationship( + 'Project', back_populates='interfaces', cascade='all', lazy='joined') + network = relationship( + 'Network', back_populates='interfaces', cascade='all', lazy='joined') + device = relationship( + 'Device', back_populates='interfaces', cascade='all', lazy='joined') class Network(Base, VariableMixin): @@ -518,7 +519,7 @@ class Network(Base, VariableMixin): cloud = relationship('Cloud', back_populates='networks') region = relationship('Region', back_populates='networks') cell = relationship('Cell', back_populates='networks') - devices = relationship('NetworkInterface', back_populates='network') + interfaces = relationship('NetworkInterface', back_populates='network') class NetworkDevice(Device): @@ -550,21 +551,3 @@ class Label(Base): self.label = label device = relationship("Device", back_populates="related_labels") - - -class AccessSecret(Base): - """Represents a secret for accessing a host. It may be shared. - For now we assume a PEM-encoded certificate that wraps the private - key. Such certs may or may not be encrypted; if encrypted, the - configuration specifies how to interact with other systems, such - as Barbican or Hashicorp Vault, to retrieve secret data to unlock - this cert. - Note that this does not include secrets such as Ansible vault - files; those are stored outside the inventory database as part of - the configuration. - """ - __tablename__ = 'access_secrets' - id = Column(Integer, primary_key=True) - cert = Column(Text) - - devices = relationship('Device', back_populates='access_secret') diff --git a/craton/tests/functional/__init__.py b/craton/tests/functional/__init__.py index 067886c..17335dd 100644 --- a/craton/tests/functional/__init__.py +++ b/craton/tests/functional/__init__.py @@ -1,16 +1,19 @@ import contextlib import copy -import docker import json +import threading + +import docker +from oslo_log import log as logging +from oslo_utils import uuidutils import requests from retrying import retry from sqlalchemy import create_engine from sqlalchemy import MetaData +from sqlalchemy.orm import sessionmaker import testtools -import threading -from oslo_log import log as logging -from oslo_utils import uuidutils +from craton.db.sqlalchemy import models LOG = logging.getLogger(__name__) @@ -161,6 +164,9 @@ def setup_database(container_ip): meta = MetaData() meta.reflect(engine) + # NOTE(sulo, jimbaker): First clean the db up for tests, and do + # our own bootstrapping to isolate all test from any external + # scripts. with contextlib.closing(engine.connect()) as conn: transaction = conn.begin() conn.execute("SET foreign_key_checks = 0") @@ -169,34 +175,35 @@ def setup_database(container_ip): conn.execute("SET foreign_key_checks = 1") transaction.commit() - # NOTE(sulo): as a part of db setup, we bootstrap user and project - # Although, project and user might have been bootstrapped externally - # we clean the db up for tests, and do our own bootstrapping to - # isolate all test from any external scripts. - projects = meta.tables['projects'] - users = meta.tables['users'] - variable_assn = meta.tables['variable_association'] + # NOTE(sulo, jimbaker): now bootstrap user and project; using the + # SA model allows us to respect the additional constraints in the + # model, vs having to duplicate logic if working against the + # database directly. + Session = sessionmaker(bind=engine) + session = Session() + project = models.Project( + name=FAKE_DATA_GEN_USERNAME, + id=FAKE_DATA_GEN_PROJECT_ID) + bootstrap_user = models.User( + project=project, + username=FAKE_DATA_GEN_BOOTSTRAP_USERNAME, + api_key=FAKE_DATA_GEN_BOOTSTRAP_TOKEN, + is_admin=True, + is_root=True) + demo_user = models.User( + project=project, + username=FAKE_DATA_GEN_USERNAME, + api_key=FAKE_DATA_GEN_TOKEN, + is_admin=True) - with contextlib.closing(engine.connect()) as conn: - transaction = conn.begin() - result = conn.execute(variable_assn.insert(), - discriminator='project') - conn.execute(projects.insert(), - name=FAKE_DATA_GEN_USERNAME, - id=FAKE_DATA_GEN_PROJECT_ID, - variable_association_id=result.inserted_primary_key[0]) - conn.execute(users.insert(), - project_id=FAKE_DATA_GEN_PROJECT_ID, - username=FAKE_DATA_GEN_USERNAME, - api_key=FAKE_DATA_GEN_TOKEN, - is_admin=True) - conn.execute(users.insert(), - project_id=FAKE_DATA_GEN_PROJECT_ID, - username=FAKE_DATA_GEN_BOOTSTRAP_USERNAME, - api_key=FAKE_DATA_GEN_BOOTSTRAP_TOKEN, - is_admin=True, - is_root=True) - transaction.commit() + session.add(project) + session.add(bootstrap_user) + session.add(demo_user) + + # NOTE(jimbaker) simple assumption: either this commit succeeds, + # or we need to fail fast - there's no recovery allowed in this + # testing setup. + session.commit() class TestCase(testtools.TestCase): diff --git a/tools/docker_run.sh b/tools/docker_run.sh index 346b377..3aa3763 100755 --- a/tools/docker_run.sh +++ b/tools/docker_run.sh @@ -20,7 +20,6 @@ mysqladmin flush-privileges ############## /craton/bin/craton-dbsync --config-file=/craton/etc/craton-api-conf.sample upgrade - #################################### # Create initial project and users # ####################################