From 36fdf739bb604c9010e61212419a9281a1e8156f Mon Sep 17 00:00:00 2001
From: Doug Hellmann <doug@doughellmann.com>
Date: Wed, 12 Nov 2014 21:09:37 +0000
Subject: [PATCH] start the oslo file handler

load the list of modules synced from the incubator into a project
---
 aeromancer/oslo/__init__.py                   |  0
 aeromancer/oslo/alembic.ini                   | 59 +++++++++++++++
 aeromancer/oslo/alembic/README                |  1 +
 aeromancer/oslo/alembic/env.py                | 72 +++++++++++++++++++
 aeromancer/oslo/alembic/script.py.mako        | 22 ++++++
 .../28d0cdc12de0_add_oslo_module_table.py     | 30 ++++++++
 aeromancer/oslo/cli.py                        | 58 +++++++++++++++
 aeromancer/oslo/handler.py                    | 51 +++++++++++++
 aeromancer/oslo/models.py                     | 21 ++++++
 setup.cfg                                     |  5 +-
 10 files changed, 318 insertions(+), 1 deletion(-)
 create mode 100644 aeromancer/oslo/__init__.py
 create mode 100644 aeromancer/oslo/alembic.ini
 create mode 100644 aeromancer/oslo/alembic/README
 create mode 100644 aeromancer/oslo/alembic/env.py
 create mode 100644 aeromancer/oslo/alembic/script.py.mako
 create mode 100644 aeromancer/oslo/alembic/versions/28d0cdc12de0_add_oslo_module_table.py
 create mode 100644 aeromancer/oslo/cli.py
 create mode 100644 aeromancer/oslo/handler.py
 create mode 100644 aeromancer/oslo/models.py

diff --git a/aeromancer/oslo/__init__.py b/aeromancer/oslo/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/aeromancer/oslo/alembic.ini b/aeromancer/oslo/alembic.ini
new file mode 100644
index 0000000..aaefd53
--- /dev/null
+++ b/aeromancer/oslo/alembic.ini
@@ -0,0 +1,59 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/aeromancer/oslo/alembic/README b/aeromancer/oslo/alembic/README
new file mode 100644
index 0000000..98e4f9c
--- /dev/null
+++ b/aeromancer/oslo/alembic/README
@@ -0,0 +1 @@
+Generic single-database configuration.
\ No newline at end of file
diff --git a/aeromancer/oslo/alembic/env.py b/aeromancer/oslo/alembic/env.py
new file mode 100644
index 0000000..18814b4
--- /dev/null
+++ b/aeromancer/oslo/alembic/env.py
@@ -0,0 +1,72 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+#fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(url=url, target_metadata=target_metadata)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    engine = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    connection = engine.connect()
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata
+    )
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.close()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()
diff --git a/aeromancer/oslo/alembic/script.py.mako b/aeromancer/oslo/alembic/script.py.mako
new file mode 100644
index 0000000..9570201
--- /dev/null
+++ b/aeromancer/oslo/alembic/script.py.mako
@@ -0,0 +1,22 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}
diff --git a/aeromancer/oslo/alembic/versions/28d0cdc12de0_add_oslo_module_table.py b/aeromancer/oslo/alembic/versions/28d0cdc12de0_add_oslo_module_table.py
new file mode 100644
index 0000000..b5984b6
--- /dev/null
+++ b/aeromancer/oslo/alembic/versions/28d0cdc12de0_add_oslo_module_table.py
@@ -0,0 +1,30 @@
+"""add oslo module table
+
+Revision ID: 28d0cdc12de0
+Revises: None
+Create Date: 2014-11-12 20:38:44.826444
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '28d0cdc12de0'
+down_revision = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table(
+        'oslo_module',
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('line_id', sa.Integer,
+                  sa.ForeignKey('line.id', name='fk_oslo_module_line_id')),
+        sa.Column('project_id', sa.Integer,
+                  sa.ForeignKey('project.id', name='fk_oslo_module_project_id')),
+        sa.Column('name', sa.String()),
+    )
+
+
+def downgrade():
+    op.drop_table('oslo_module')
diff --git a/aeromancer/oslo/cli.py b/aeromancer/oslo/cli.py
new file mode 100644
index 0000000..75e9e91
--- /dev/null
+++ b/aeromancer/oslo/cli.py
@@ -0,0 +1,58 @@
+import logging
+import os
+
+from aeromancer.db import models
+from aeromancer.oslo import models as oslo_models
+from aeromancer import project
+from aeromancer import utils
+
+from cliff.lister import Lister
+
+from sqlalchemy import distinct
+from sqlalchemy.orm import aliased
+
+
+class List(Lister):
+    """List the Oslo modules used by a project"""
+
+    log = logging.getLogger(__name__)
+
+    def get_parser(self, prog_name):
+        parser = super(List, self).get_parser(prog_name)
+        parser.add_argument(
+            'project',
+            help=('project directory name under the project root, '
+                  'for example: "stackforge/aeromancer"'),
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        session = self.app.get_db_session()
+        query = session.query(oslo_models.Module).join(models.Project).filter(
+            models.Project.name == parsed_args.project
+        ).order_by(oslo_models.Module.name)
+        return (('Name',),
+                ((r.name,)
+                 for r in query.all()))
+
+
+class Uses(Lister):
+    """List the projects that use the Oslo module"""
+
+    log = logging.getLogger(__name__)
+
+    def get_parser(self, prog_name):
+        parser = super(Uses, self).get_parser(prog_name)
+        parser.add_argument(
+            'module',
+            help='the module name',
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        session = self.app.get_db_session()
+        query = session.query(oslo_models.Module).join(models.Project).filter(
+            oslo_models.Module.name == parsed_args.module
+        ).order_by(models.Project.name)
+        return (('Name',),
+                ((r.project.name,) for r in query.all()))
diff --git a/aeromancer/oslo/handler.py b/aeromancer/oslo/handler.py
new file mode 100644
index 0000000..cbeae95
--- /dev/null
+++ b/aeromancer/oslo/handler.py
@@ -0,0 +1,51 @@
+import logging
+
+import pkg_resources
+
+from aeromancer.db import models as models
+from aeromancer.filehandler import base
+from aeromancer.oslo import models as oslo_models
+
+LOG = logging.getLogger(__name__)
+
+
+def read_sync_file(file_obj):
+    for line in file_obj.lines:
+        text = line.content.strip()
+        if not text or text.startswith('#'):
+            continue
+        if not text.startswith('module'):
+            continue
+        text = text[len('module'):]
+        text = text.lstrip('= ')
+        modules = text.split(',')
+        for m in modules:
+            yield m.strip(), line
+
+
+class OsloSyncHandler(base.FileHandler):
+
+    INTERESTING_PATTERNS = [
+        'openstack-common.conf',
+    ]
+
+    def process_file(self, session, file_obj):
+        LOG.info('loading Oslo settings from %s', file_obj.project_path)
+        parent_project = file_obj.project
+        for module_name, line in read_sync_file(file_obj):
+            LOG.debug('module: %s', module_name)
+            new_r = oslo_models.Module(
+                name=module_name,
+                line=line,
+                project=parent_project,
+            )
+            session.add(new_r)
+
+    def delete_data_for_file(self, session, file_obj):
+        LOG.debug('deleting Oslo modules from %r', file_obj.path)
+        query = session.query(oslo_models.Module).join(models.Line).filter(
+            models.Line.file_id == file_obj.id
+        )
+        for r in query.all():
+            session.delete(r)
+        return
diff --git a/aeromancer/oslo/models.py b/aeromancer/oslo/models.py
new file mode 100644
index 0000000..81ecac9
--- /dev/null
+++ b/aeromancer/oslo/models.py
@@ -0,0 +1,21 @@
+from sqlalchemy import Column, Integer, String, ForeignKey
+from sqlalchemy.orm import relationship, backref
+
+from aeromancer.db import models
+
+
+class Module(models.Base):
+    __tablename__ = 'oslo_module'
+    id = Column(Integer, primary_key=True)
+    name = Column(String, nullable=False)
+    line_id = Column(Integer, ForeignKey('line.id'))
+    line = relationship(
+        models.Line,
+        uselist=False,
+        single_parent=True,
+    )
+    project_id = Column(Integer, ForeignKey('project.id'))
+    project = relationship(
+        models.Project,
+        backref='oslo_modules',
+    )
diff --git a/setup.cfg b/setup.cfg
index 320b635..3ab1ea0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -58,8 +58,11 @@ aeromancer.cli =
     requirements list = aeromancer.requirements.cli:List
     requirements outdated = aeromancer.requirements.cli:Outdated
     requirements unused = aeromancer.requirements.cli:Unused
-    what uses = aeromancer.requirements.cli:Uses
+    requirements uses = aeromancer.requirements.cli:Uses
+	oslo list = aeromancer.oslo.cli:List
+	oslo uses = aeromancer.oslo.cli:Uses
 
 aeromancer.filehandler =
     requirements = aeromancer.requirements.handler:RequirementsHandler
     global_requirements = aeromancer.requirements.handler:GlobalRequirementsHandler
+    oslo_sync = aeromancer.oslo.handler:OsloSyncHandler