Added basic implementation of extension
* Database model and migration script * CRUD for database entities * GET REST call for single entity and for collection Change-Id: I0446983d9af89c316f6754168123de3da6ed10ff
This commit is contained in:
parent
1321d76757
commit
cfb660f76d
|
@ -1 +1,2 @@
|
||||||
# fuel-external-git
|
# fuel-external-git
|
||||||
|
Nailgun extension that generates deployment data based on configuration files published in external git repository
|
||||||
|
|
|
@ -0,0 +1,68 @@
|
||||||
|
# A generic, single database configuration.
|
||||||
|
|
||||||
|
[alembic]
|
||||||
|
# path to migration scripts
|
||||||
|
script_location = migrations/
|
||||||
|
|
||||||
|
# template used to generate migration files
|
||||||
|
# file_template = %%(rev)s_%%(slug)s
|
||||||
|
|
||||||
|
# max length of characters to apply to the
|
||||||
|
# "slug" field
|
||||||
|
#truncate_slug_length = 40
|
||||||
|
|
||||||
|
# set to 'true' to run the environment during
|
||||||
|
# the 'revision' command, regardless of autogenerate
|
||||||
|
# revision_environment = false
|
||||||
|
|
||||||
|
# set to 'true' to allow .pyc and .pyo files without
|
||||||
|
# a source .py file to be detected as revisions in the
|
||||||
|
# versions/ directory
|
||||||
|
# sourceless = false
|
||||||
|
|
||||||
|
# version location specification; this defaults
|
||||||
|
# to migrations//versions. When using multiple version
|
||||||
|
# directories, initial revisions must be specified with --version-path
|
||||||
|
# version_locations = %(here)s/bar %(here)s/bat migrations//versions
|
||||||
|
|
||||||
|
# the output encoding used when revision files
|
||||||
|
# are written from script.py.mako
|
||||||
|
# output_encoding = utf-8
|
||||||
|
|
||||||
|
sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||||
|
|
||||||
|
|
||||||
|
# Logging configuration
|
||||||
|
[loggers]
|
||||||
|
keys = root,sqlalchemy,alembic
|
||||||
|
|
||||||
|
[handlers]
|
||||||
|
keys = console
|
||||||
|
|
||||||
|
[formatters]
|
||||||
|
keys = generic
|
||||||
|
|
||||||
|
[logger_root]
|
||||||
|
level = WARN
|
||||||
|
handlers = console
|
||||||
|
qualname =
|
||||||
|
|
||||||
|
[logger_sqlalchemy]
|
||||||
|
level = WARN
|
||||||
|
handlers =
|
||||||
|
qualname = sqlalchemy.engine
|
||||||
|
|
||||||
|
[logger_alembic]
|
||||||
|
level = INFO
|
||||||
|
handlers =
|
||||||
|
qualname = alembic
|
||||||
|
|
||||||
|
[handler_console]
|
||||||
|
class = StreamHandler
|
||||||
|
args = (sys.stderr,)
|
||||||
|
level = NOTSET
|
||||||
|
formatter = generic
|
||||||
|
|
||||||
|
[formatter_generic]
|
||||||
|
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||||
|
datefmt = %H:%M:%S
|
|
@ -0,0 +1 @@
|
||||||
|
REPOS_DIR = '/var/lib/fuel_repos'
|
|
@ -0,0 +1,32 @@
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from nailgun import objects
|
||||||
|
from nailgun.logger import logger
|
||||||
|
|
||||||
|
from nailgun.extensions import BaseExtension
|
||||||
|
from nailgun.extensions import BasePipeline
|
||||||
|
|
||||||
|
from fuel_external_git import handlers
|
||||||
|
|
||||||
|
|
||||||
|
class OpenStackConfigPipeline(BasePipeline):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class ExternalGit(BaseExtension):
|
||||||
|
name = 'fuel_external_git'
|
||||||
|
version = '1.0.0'
|
||||||
|
description = 'Nailgun extension which uses git repo for config files'
|
||||||
|
|
||||||
|
urls = [{'uri': r'/clusters/git-repos/?$',
|
||||||
|
'handler': handlers.GitRepoCollectionHandler},
|
||||||
|
{'uri': r'/clusters/(?P<cluster_id>\d+)/git-repos/(?P<obj_id>\d+)?$',
|
||||||
|
'handler': handlers.GitRepoHandler}]
|
||||||
|
|
||||||
|
data_pipelines = [
|
||||||
|
OpenStackConfigPipeline,
|
||||||
|
]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def alembic_migrations_path(cls):
|
||||||
|
return os.path.join(os.path.dirname(__file__), 'migrations')
|
|
@ -0,0 +1,68 @@
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import web
|
||||||
|
|
||||||
|
from nailgun.api.v1.handlers.base import SingleHandler, CollectionHandler
|
||||||
|
from nailgun.api.v1.handlers.base import content
|
||||||
|
from nailgun.logger import logger
|
||||||
|
from nailgun import objects
|
||||||
|
|
||||||
|
from fuel_external_git.objects import GitRepo, GitRepoCollection
|
||||||
|
from git import Repo
|
||||||
|
|
||||||
|
REPOS_DIR = '/var/lib/fuel_repos'
|
||||||
|
|
||||||
|
class GitRepoCollectionHandler(CollectionHandler):
|
||||||
|
collection = GitRepoCollection
|
||||||
|
|
||||||
|
class GitRepoHandler(SingleHandler):
|
||||||
|
single = GitRepo
|
||||||
|
|
||||||
|
def GET(self, cluster_id, obj_id):
|
||||||
|
""":returns: JSONized REST object.
|
||||||
|
|
||||||
|
:http: * 200 (OK)
|
||||||
|
* 404 (dashboard entry not found in db)
|
||||||
|
"""
|
||||||
|
self.get_object_or_404(objects.Cluster, cluster_id)
|
||||||
|
|
||||||
|
obj = self.get_object_or_404(self.single, obj_id)
|
||||||
|
return self.single.to_json(obj)
|
||||||
|
|
||||||
|
@content
|
||||||
|
def PUT(self, cluster_id, obj_id):
|
||||||
|
""":returns: JSONized REST object.
|
||||||
|
|
||||||
|
:http: * 200 (OK)
|
||||||
|
* 400 (invalid object data specified)
|
||||||
|
* 404 (object not found in db)
|
||||||
|
"""
|
||||||
|
obj = self.get_object_or_404(self.single, obj_id)
|
||||||
|
|
||||||
|
data = self.checked_data(
|
||||||
|
self.validator.validate_update,
|
||||||
|
instance=obj
|
||||||
|
)
|
||||||
|
self.single.update(obj, data)
|
||||||
|
return self.single.to_json(obj)
|
||||||
|
|
||||||
|
def PATCH(self, cluster_id, obj_id):
|
||||||
|
""":returns: JSONized REST object.
|
||||||
|
|
||||||
|
:http: * 200 (OK)
|
||||||
|
* 400 (invalid object data specified)
|
||||||
|
* 404 (object not found in db)
|
||||||
|
"""
|
||||||
|
return self.PUT(cluster_id, obj_id)
|
||||||
|
|
||||||
|
@content
|
||||||
|
def DELETE(self, cluster_id, obj_id):
|
||||||
|
""":returns: JSONized REST object.
|
||||||
|
|
||||||
|
:http: * 204 (OK)
|
||||||
|
* 404 (object not found in db)
|
||||||
|
"""
|
||||||
|
d_e = self.get_object_or_404(self.single, obj_id)
|
||||||
|
self.single.delete(d_e)
|
||||||
|
raise self.http(204)
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
Generic single-database configuration.
|
|
@ -0,0 +1,73 @@
|
||||||
|
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import logging.config
|
||||||
|
|
||||||
|
from alembic import context
|
||||||
|
import sqlalchemy
|
||||||
|
|
||||||
|
#from tuning_box import db
|
||||||
|
|
||||||
|
config = context.config
|
||||||
|
if config.get_main_option('table_prefix') is None:
|
||||||
|
config.set_main_option('table_prefix', '')
|
||||||
|
if config.config_file_name:
|
||||||
|
logging.config.fileConfig(config.config_file_name)
|
||||||
|
#target_metadata = db.db.metadata
|
||||||
|
target_metadata = None
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_offline():
|
||||||
|
"""Run migrations in 'offline' mode.
|
||||||
|
This configures the context with just a URL
|
||||||
|
and not an Engine, though an Engine is acceptable
|
||||||
|
here as well. By skipping the Engine creation
|
||||||
|
we don't even need a DBAPI to be available.
|
||||||
|
Calls to context.execute() here emit the given string to the
|
||||||
|
script output.
|
||||||
|
"""
|
||||||
|
context.configure(
|
||||||
|
url=config.get_main_option('sqlalchemy.url'),
|
||||||
|
version_table=config.get_main_option('version_table'),
|
||||||
|
literal_binds=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_online():
|
||||||
|
"""Run migrations in 'online' mode.
|
||||||
|
In this scenario we need to create an Engine
|
||||||
|
and associate a connection with the context.
|
||||||
|
"""
|
||||||
|
connectable = sqlalchemy.engine_from_config(
|
||||||
|
config.get_section(config.config_ini_section),
|
||||||
|
prefix='sqlalchemy.',
|
||||||
|
poolclass=sqlalchemy.pool.NullPool,
|
||||||
|
)
|
||||||
|
|
||||||
|
with connectable.connect() as connection:
|
||||||
|
context.configure(
|
||||||
|
connection=connection,
|
||||||
|
target_metadata=target_metadata,
|
||||||
|
version_table=config.get_main_option('version_table'),
|
||||||
|
)
|
||||||
|
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
if context.is_offline_mode():
|
||||||
|
run_migrations_offline()
|
||||||
|
else:
|
||||||
|
run_migrations_online()
|
|
@ -0,0 +1,24 @@
|
||||||
|
"""${message}
|
||||||
|
|
||||||
|
Revision ID: ${up_revision}
|
||||||
|
Revises: ${down_revision | comma,n}
|
||||||
|
Create Date: ${create_date}
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = ${repr(up_revision)}
|
||||||
|
down_revision = ${repr(down_revision)}
|
||||||
|
branch_labels = ${repr(branch_labels)}
|
||||||
|
depends_on = ${repr(depends_on)}
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
${imports if imports else ""}
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
${upgrades if upgrades else "pass"}
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
${downgrades if downgrades else "pass"}
|
|
@ -0,0 +1,40 @@
|
||||||
|
"""Init
|
||||||
|
|
||||||
|
Revision ID: e3b840e64e53
|
||||||
|
Revises:
|
||||||
|
Create Date: 2016-08-09 16:59:36.504052
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = 'e3b840e64e53'
|
||||||
|
down_revision = None
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
from alembic import context
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects import postgresql as psql
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
table_prefix = context.config.get_main_option('table_prefix')
|
||||||
|
op.create_table(
|
||||||
|
table_prefix + 'repos',
|
||||||
|
sa.Column('id', sa.Integer(), nullable=False, primary_key=True),
|
||||||
|
sa.Column('repo_name', sa.Unicode(100), nullable=False)
|
||||||
|
sa.Column('env_id', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('git_url', sa.String(255),
|
||||||
|
server_default='', nullable=False),
|
||||||
|
sa.Column('ref', sa.String(255),
|
||||||
|
server_default='', nullable=False))
|
||||||
|
sa.Column('user_key', sa.String(255))
|
||||||
|
server_default='', nullable=False))
|
||||||
|
sa.UniqueConstraint('env_id', name='_env_id_unique')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
table_prefix = context.config.get_main_option('table_prefix')
|
||||||
|
op.drop_table(table_prefix + 'repos')
|
|
@ -0,0 +1,19 @@
|
||||||
|
from sqlalchemy import Column
|
||||||
|
from sqlalchemy import Integer
|
||||||
|
from sqlalchemy import String
|
||||||
|
from sqlalchemy import UnicodeText
|
||||||
|
|
||||||
|
from sqlalchemy.dialects import postgresql as psql
|
||||||
|
|
||||||
|
from nailgun.db import db
|
||||||
|
from nailgun.errors import errors
|
||||||
|
from nailgun.db.sqlalchemy.models.base import Base
|
||||||
|
|
||||||
|
class GitRepo(Base):
|
||||||
|
__tablename__ = 'fuel_external_git_repos'
|
||||||
|
id = Column(Integer, primary_key=True)
|
||||||
|
repo_name = Column(UnicodeText, nullable=False)
|
||||||
|
env_id = Column(Integer, unique=True, nullable=False)
|
||||||
|
git_url = Column(String(255), default='', server_default='', nullable=False)
|
||||||
|
ref = Column(String(255), default='', server_default='', nullable=False)
|
||||||
|
user_key = Column(String(255), default='', server_default='', nullable=False)
|
|
@ -0,0 +1,61 @@
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
from nailgun.db import db
|
||||||
|
from nailgun.objects import NailgunObject, NailgunCollection
|
||||||
|
from nailgun.objects.serializers.base import BasicSerializer
|
||||||
|
|
||||||
|
from git import Repo
|
||||||
|
from git import exc
|
||||||
|
|
||||||
|
from fuel_external_git.models import GitRepo
|
||||||
|
from fuel_external_git import const
|
||||||
|
|
||||||
|
class GitRepoSerializer(BasicSerializer):
|
||||||
|
fields = (
|
||||||
|
"id",
|
||||||
|
"repo_name",
|
||||||
|
"env_id",
|
||||||
|
"git_url",
|
||||||
|
"ref",
|
||||||
|
"user_key"
|
||||||
|
)
|
||||||
|
|
||||||
|
class GitRepo(NailgunObject):
|
||||||
|
model = GitRepo
|
||||||
|
serializer = GitRepoSerializer
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_by_cluster_id(self, cluster_id):
|
||||||
|
instance = db().query(self.model).\
|
||||||
|
filter(self.model.env_id == cluster_id).\
|
||||||
|
first()
|
||||||
|
if instance is not None:
|
||||||
|
try:
|
||||||
|
instance.repo = Repo(os.path.join(const.REPOS_DIR, instance.repo_name))
|
||||||
|
except exc.NoSuchPathError:
|
||||||
|
# TODO(dukov) Put some logging here
|
||||||
|
instance.repo = GitRepo.clone(instance.git_url)
|
||||||
|
return instance
|
||||||
|
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def create(self, data):
|
||||||
|
repo_path = os.path.join(const.REPOS_DIR, data['repo_name'])
|
||||||
|
if os.path.exists(repo_path):
|
||||||
|
shutil.rmtree(repo_path)
|
||||||
|
repo = Repo.clone_from(data['git_url'], repo_path)
|
||||||
|
instance = super(GitRepo, self).create(data)
|
||||||
|
instance.repo = repo
|
||||||
|
return instance
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def checkout(self, instance):
|
||||||
|
commit = instance.repo.remotes.origin.fetch(refspec=instance.ref)[0].commit
|
||||||
|
instance.repo.head.reference = commit
|
||||||
|
|
||||||
|
def remove_repo(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class GitRepoCollection(NailgunCollection):
|
||||||
|
single = GitRepo
|
||||||
|
|
|
@ -0,0 +1,61 @@
|
||||||
|
import os
|
||||||
|
|
||||||
|
from setuptools import setup
|
||||||
|
from setuptools.command.install import install
|
||||||
|
|
||||||
|
from nailgun.db import db
|
||||||
|
from nailgun.db.sqlalchemy.models import Cluster
|
||||||
|
from nailgun.db.sqlalchemy.models import Release
|
||||||
|
|
||||||
|
def package_files(directory):
|
||||||
|
paths = []
|
||||||
|
for (path, directories, filenames) in os.walk(directory):
|
||||||
|
for filename in filenames:
|
||||||
|
paths.append(os.path.join('..', path, filename))
|
||||||
|
return paths
|
||||||
|
|
||||||
|
extra_files = package_files('fuel_external_git/migrations')
|
||||||
|
|
||||||
|
class ExtInstall(install):
|
||||||
|
@classmethod
|
||||||
|
def _set_extensions(self, nailgun_objects):
|
||||||
|
for obj, extensions in nailgun_objects.items():
|
||||||
|
extensions.append(u'fuel_external_git')
|
||||||
|
extensions = list(set(extensions))
|
||||||
|
obj.extensions = extensions
|
||||||
|
db().flush()
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
install.run(self)
|
||||||
|
clusters = {cl: cl['extensions'] for cl in db().query(Cluster).all()}
|
||||||
|
releases = {rl: rl['extensions'] for rl in db().query(Release).all()}
|
||||||
|
ExtInstall._set_extensions(clusters)
|
||||||
|
ExtInstall._set_extensions(releases)
|
||||||
|
db().commit()
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name='fuel_external_git',
|
||||||
|
version='1.0',
|
||||||
|
description='Nailgun extension which uses git repo for config files',
|
||||||
|
author='Dmitry Ukov',
|
||||||
|
author_email='dukov@mirantis.com',
|
||||||
|
url='http://example.com',
|
||||||
|
classifiers=['Development Status :: 3 - Alpha',
|
||||||
|
'License :: OSI Approved :: Apache Software License',
|
||||||
|
'Programming Language :: Python',
|
||||||
|
'Programming Language :: Python :: 2',
|
||||||
|
'Environment :: Console',
|
||||||
|
],
|
||||||
|
packages=['fuel_external_git'],
|
||||||
|
package_data={'fuel_external_git': extra_files},
|
||||||
|
cmdclass={'install': ExtInstall},
|
||||||
|
entry_points={
|
||||||
|
'nailgun.extensions': [
|
||||||
|
'fuel_external_git = fuel_external_git.extension:ExternalGit',
|
||||||
|
],
|
||||||
|
'fuelclient': [
|
||||||
|
'set-git-repo = fuel_external_git.fuelclient:Generate',
|
||||||
|
]
|
||||||
|
},
|
||||||
|
zip_safe=False,
|
||||||
|
)
|
Loading…
Reference in New Issue