Added basic implementation of extension

* Database model and migration script
* CRUD for database entities
* GET REST call for single entity and for collection

Change-Id: I0446983d9af89c316f6754168123de3da6ed10ff

README.md

@@ -1 +1,2 @@
-# fuel-external-git
+# fuel-external-git
+Nailgun extension that generates deployment data based on configuration files published in external git repository

fuel_external_git/alembic.ini

@@ -0,0 +1,68 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = migrations/
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to migrations//versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat migrations//versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

fuel_external_git/const.py

@@ -0,0 +1 @@
+REPOS_DIR = '/var/lib/fuel_repos'

fuel_external_git/extension.py

@@ -0,0 +1,32 @@
+import os
+import logging
+
+from nailgun import objects
+from nailgun.logger import logger
+
+from nailgun.extensions import BaseExtension
+from nailgun.extensions import BasePipeline
+
+from fuel_external_git import handlers
+
+
+class OpenStackConfigPipeline(BasePipeline):
+    pass
+
+class ExternalGit(BaseExtension):
+    name = 'fuel_external_git'
+    version = '1.0.0'
+    description = 'Nailgun extension which uses git repo for config files'
+
+    urls = [{'uri': r'/clusters/git-repos/?$',
+             'handler': handlers.GitRepoCollectionHandler},
+            {'uri': r'/clusters/(?P<cluster_id>\d+)/git-repos/(?P<obj_id>\d+)?$',
+             'handler': handlers.GitRepoHandler}]
+
+    data_pipelines = [
+        OpenStackConfigPipeline,
+    ]
+
+    @classmethod
+    def alembic_migrations_path(cls):
+        return os.path.join(os.path.dirname(__file__), 'migrations')

fuel_external_git/handlers.py

@@ -0,0 +1,68 @@
+import json
+import os
+import web
+
+from nailgun.api.v1.handlers.base import SingleHandler, CollectionHandler
+from nailgun.api.v1.handlers.base import content
+from nailgun.logger import logger
+from nailgun import objects
+
+from fuel_external_git.objects import GitRepo, GitRepoCollection 
+from git import Repo
+
+REPOS_DIR = '/var/lib/fuel_repos'
+
+class GitRepoCollectionHandler(CollectionHandler):
+    collection = GitRepoCollection
+
+class GitRepoHandler(SingleHandler):
+    single = GitRepo
+
+    def GET(self, cluster_id, obj_id):
+        """:returns: JSONized REST object.
+
+        :http: * 200 (OK)
+               * 404 (dashboard entry not found in db)
+        """
+        self.get_object_or_404(objects.Cluster, cluster_id)
+
+        obj = self.get_object_or_404(self.single, obj_id)
+        return self.single.to_json(obj)
+
+    @content
+    def PUT(self, cluster_id, obj_id):
+        """:returns: JSONized REST object.
+
+        :http: * 200 (OK)
+               * 400 (invalid object data specified)
+               * 404 (object not found in db)
+        """
+        obj = self.get_object_or_404(self.single, obj_id)
+
+        data = self.checked_data(
+            self.validator.validate_update,
+            instance=obj
+        )
+        self.single.update(obj, data)
+        return self.single.to_json(obj)
+
+    def PATCH(self, cluster_id, obj_id):
+        """:returns: JSONized REST object.
+
+        :http: * 200 (OK)
+               * 400 (invalid object data specified)
+               * 404 (object not found in db)
+        """
+        return self.PUT(cluster_id, obj_id)
+
+    @content
+    def DELETE(self, cluster_id, obj_id):
+        """:returns: JSONized REST object.
+
+        :http: * 204 (OK)
+               * 404 (object not found in db)
+        """
+        d_e = self.get_object_or_404(self.single, obj_id)
+        self.single.delete(d_e)
+        raise self.http(204)
+

fuel_external_git/migrations/README

@@ -0,0 +1 @@
+Generic single-database configuration.

fuel_external_git/migrations/env.py

@@ -0,0 +1,73 @@
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging.config
+
+from alembic import context
+import sqlalchemy
+
+#from tuning_box import db
+
+config = context.config
+if config.get_main_option('table_prefix') is None:
+    config.set_main_option('table_prefix', '')
+if config.config_file_name:
+    logging.config.fileConfig(config.config_file_name)
+#target_metadata = db.db.metadata
+target_metadata = None
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+    Calls to context.execute() here emit the given string to the
+    script output.
+    """
+    context.configure(
+        url=config.get_main_option('sqlalchemy.url'),
+        version_table=config.get_main_option('version_table'),
+        literal_binds=True,
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+    """
+    connectable = sqlalchemy.engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=sqlalchemy.pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            version_table=config.get_main_option('version_table'),
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

fuel_external_git/migrations/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

fuel_external_git/migrations/versions/e3b840e64e53_init.py

@@ -0,0 +1,40 @@
+"""Init
+
+Revision ID: e3b840e64e53
+Revises: 
+Create Date: 2016-08-09 16:59:36.504052
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'e3b840e64e53'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+from alembic import context
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql as psql
+
+
+def upgrade():
+    table_prefix = context.config.get_main_option('table_prefix')
+    op.create_table(
+        table_prefix + 'repos',
+        sa.Column('id', sa.Integer(), nullable=False, primary_key=True),
+        sa.Column('repo_name', sa.Unicode(100), nullable=False)
+        sa.Column('env_id', sa.Integer(), nullable=False),
+        sa.Column('git_url', sa.String(255),
+                  server_default='', nullable=False),
+        sa.Column('ref', sa.String(255),
+                  server_default='', nullable=False))
+        sa.Column('user_key', sa.String(255))
+                  server_default='', nullable=False))
+        sa.UniqueConstraint('env_id', name='_env_id_unique')
+        
+
+
+def downgrade():
+    table_prefix = context.config.get_main_option('table_prefix')
+    op.drop_table(table_prefix + 'repos')

fuel_external_git/models.py

@@ -0,0 +1,19 @@
+from sqlalchemy import Column
+from sqlalchemy import Integer
+from sqlalchemy import String
+from sqlalchemy import UnicodeText
+
+from sqlalchemy.dialects import postgresql as psql
+
+from nailgun.db import db
+from nailgun.errors import errors
+from nailgun.db.sqlalchemy.models.base import Base
+
+class GitRepo(Base):
+    __tablename__ = 'fuel_external_git_repos'
+    id = Column(Integer, primary_key=True)
+    repo_name = Column(UnicodeText, nullable=False)
+    env_id = Column(Integer, unique=True, nullable=False)
+    git_url = Column(String(255), default='', server_default='', nullable=False)
+    ref = Column(String(255), default='', server_default='', nullable=False)
+    user_key = Column(String(255), default='', server_default='', nullable=False)

fuel_external_git/objects.py

@@ -0,0 +1,61 @@
+import os
+import shutil
+from nailgun.db import db
+from nailgun.objects import NailgunObject, NailgunCollection
+from nailgun.objects.serializers.base import BasicSerializer
+
+from git import Repo
+from git import exc
+
+from fuel_external_git.models import GitRepo 
+from fuel_external_git import const
+
+class GitRepoSerializer(BasicSerializer):
+    fields = (
+        "id",
+        "repo_name",
+        "env_id",
+        "git_url",
+        "ref",
+        "user_key"
+    )
+
+class GitRepo(NailgunObject):
+    model = GitRepo
+    serializer = GitRepoSerializer
+
+    @classmethod
+    def get_by_cluster_id(self, cluster_id):
+        instance = db().query(self.model).\
+                              filter(self.model.env_id == cluster_id).\
+                              first()
+        if instance is not None:
+            try:
+                instance.repo = Repo(os.path.join(const.REPOS_DIR, instance.repo_name))
+            except exc.NoSuchPathError:
+                # TODO(dukov) Put some logging here
+                instance.repo = GitRepo.clone(instance.git_url)
+        return instance
+           
+    
+    @classmethod
+    def create(self, data):
+        repo_path = os.path.join(const.REPOS_DIR, data['repo_name'])
+        if os.path.exists(repo_path):
+            shutil.rmtree(repo_path)
+        repo = Repo.clone_from(data['git_url'], repo_path)
+        instance = super(GitRepo, self).create(data)
+        instance.repo = repo
+        return instance
+
+    @classmethod
+    def checkout(self, instance):
+        commit = instance.repo.remotes.origin.fetch(refspec=instance.ref)[0].commit
+        instance.repo.head.reference = commit
+        
+    def remove_repo(self):
+        pass
+
+class GitRepoCollection(NailgunCollection):
+    single = GitRepo
+

setup.py

@@ -0,0 +1,61 @@
+import os
+
+from setuptools import setup
+from setuptools.command.install import install
+
+from nailgun.db import db
+from nailgun.db.sqlalchemy.models import Cluster
+from nailgun.db.sqlalchemy.models import Release
+
+def package_files(directory):
+    paths = []
+    for (path, directories, filenames) in os.walk(directory):
+        for filename in filenames:
+            paths.append(os.path.join('..', path, filename))
+    return paths
+
+extra_files = package_files('fuel_external_git/migrations')
+
+class ExtInstall(install):
+    @classmethod
+    def _set_extensions(self, nailgun_objects):
+        for obj, extensions in nailgun_objects.items():
+            extensions.append(u'fuel_external_git')
+            extensions = list(set(extensions))
+            obj.extensions = extensions
+            db().flush()
+
+    def run(self):
+        install.run(self)
+        clusters = {cl: cl['extensions'] for cl in db().query(Cluster).all()}
+        releases = {rl: rl['extensions'] for rl in db().query(Release).all()}
+        ExtInstall._set_extensions(clusters)
+        ExtInstall._set_extensions(releases)
+        db().commit()
+
+setup(
+       name='fuel_external_git',
+       version='1.0',
+       description='Nailgun extension which uses git repo for config files',
+       author='Dmitry Ukov',
+       author_email='dukov@mirantis.com',
+       url='http://example.com',
+       classifiers=['Development Status :: 3 - Alpha',
+                    'License :: OSI Approved :: Apache Software License',
+                    'Programming Language :: Python',
+                    'Programming Language :: Python :: 2',
+                    'Environment :: Console',
+                    ],
+       packages=['fuel_external_git'],
+       package_data={'fuel_external_git': extra_files},
+       cmdclass={'install': ExtInstall},
+       entry_points={
+          'nailgun.extensions': [
+             'fuel_external_git = fuel_external_git.extension:ExternalGit',
+           ],
+           'fuelclient': [
+              'set-git-repo = fuel_external_git.fuelclient:Generate',
+           ]
+       },
+       zip_safe=False,
+)