add requirements handling
Add support for file-specific plugins, including schema changes and models. Implement scanner for requirements list files and commands to list requirements for a project and find projects using a specific requirement.
This commit is contained in:
parent
324a11f69f
commit
581755045a
|
@ -1,24 +1,57 @@
|
|||
import logging
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic import config
|
||||
from alembic import command
|
||||
from alembic.environment import EnvironmentContext
|
||||
from alembic.script import ScriptDirectory
|
||||
from alembic import environment
|
||||
from alembic import script
|
||||
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from aeromancer.db import connect
|
||||
from aeromancer import filehandler
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_migrations():
|
||||
config = Config()
|
||||
config.set_main_option("script_location", "aeromancer.db:alembic")
|
||||
def _run_migrations_in_location(location):
|
||||
LOG.debug('loading migrations from %s', location)
|
||||
|
||||
url = connect.get_url()
|
||||
config.set_main_option("sqlalchemy.url", url)
|
||||
command.upgrade(config, 'head')
|
||||
# NOTE(dhellmann): Load migration settings from the plugins for
|
||||
# processing special types of files, and run them.
|
||||
|
||||
# We need a unique version_table for each set of migrations.
|
||||
version_table = location.replace('.', '_') + '_versions'
|
||||
|
||||
# Modified version of alembic.command.upgrade().
|
||||
# command.upgrade(cfg, 'head')
|
||||
revision = 'head'
|
||||
|
||||
cfg = config.Config()
|
||||
cfg.set_main_option('script_location', location + ':alembic')
|
||||
cfg.set_main_option("sqlalchemy.url", url)
|
||||
|
||||
script_dir = script.ScriptDirectory.from_config(cfg)
|
||||
|
||||
def upgrade(rev, context):
|
||||
return script_dir._upgrade_revs(revision, rev)
|
||||
|
||||
with environment.EnvironmentContext(
|
||||
cfg,
|
||||
script_dir,
|
||||
fn=upgrade,
|
||||
as_sql=False,
|
||||
starting_rev=None,
|
||||
destination_rev=revision,
|
||||
tag=None,
|
||||
version_table=version_table,
|
||||
):
|
||||
script_dir.run_env()
|
||||
|
||||
|
||||
def run_migrations():
|
||||
_run_migrations_in_location("aeromancer.db")
|
||||
file_handlers = filehandler.load_handlers()
|
||||
for fh in file_handlers:
|
||||
_run_migrations_in_location(fh.entry_point.module_name)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
@ -0,0 +1,13 @@
|
|||
from stevedore import extension
|
||||
|
||||
|
||||
def _load_error_handler(*args, **kwds):
|
||||
raise
|
||||
|
||||
|
||||
def load_handlers():
|
||||
return extension.ExtensionManager(
|
||||
'aeromancer.filehandler',
|
||||
invoke_on_load=True,
|
||||
on_load_failure_callback=_load_error_handler,
|
||||
)
|
|
@ -0,0 +1,22 @@
|
|||
import abc
|
||||
import fnmatch
|
||||
|
||||
|
||||
class FileHandler(object):
|
||||
__metaclass__ = abc.ABCMeta
|
||||
|
||||
INTERESTING_PATTERNS = []
|
||||
|
||||
def supports_file(self, file_obj):
|
||||
"""Does this plugin want to process the file?
|
||||
"""
|
||||
return any(fnmatch.fnmatch(file_obj.path, ip)
|
||||
for ip in self.INTERESTING_PATTERNS)
|
||||
|
||||
@abc.abstractmethod
|
||||
def process_file(self, session, file_obj):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def delete_data_for_file(self, session, file_obj):
|
||||
return
|
|
@ -6,15 +6,28 @@ import logging
|
|||
import os
|
||||
import subprocess
|
||||
|
||||
from aeromancer.db.models import Project, File, Line
|
||||
from aeromancer import utils
|
||||
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from aeromancer.db.models import Project, File, Line
|
||||
from aeromancer import filehandler
|
||||
from aeromancer import utils
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _delete_filehandler_data_from_project(session, proj_obj):
|
||||
# We have to explicitly have the handlers delete their data
|
||||
# because the parent-child relationship of the tables is reversed
|
||||
# because the plugins define the relationships.
|
||||
file_handlers = filehandler.load_handlers()
|
||||
LOG.debug('deleting plugin data for %s', proj_obj.name)
|
||||
for file_obj in proj_obj.files:
|
||||
for fh in file_handlers:
|
||||
if fh.obj.supports_file(file_obj):
|
||||
fh.obj.delete_data_for_file(session, file_obj)
|
||||
|
||||
|
||||
def add_or_update(session, name, path):
|
||||
"""Create a new project definition or update an existing one"""
|
||||
query = session.query(Project).filter(Project.name == name)
|
||||
|
@ -26,7 +39,6 @@ def add_or_update(session, name, path):
|
|||
proj_obj = Project(name=name, path=path)
|
||||
LOG.info('adding project %s from %s', name, path)
|
||||
session.add(proj_obj)
|
||||
|
||||
update(session, proj_obj)
|
||||
return proj_obj
|
||||
|
||||
|
@ -39,6 +51,7 @@ def remove(session, name):
|
|||
LOG.info('removing project %s', name)
|
||||
except NoResultFound:
|
||||
return
|
||||
_delete_filehandler_data_from_project(session, proj_obj)
|
||||
session.delete(proj_obj)
|
||||
|
||||
|
||||
|
@ -70,13 +83,16 @@ _DO_NOT_READ = [
|
|||
|
||||
def _update_project_files(session, proj_obj):
|
||||
"""Update the files stored for each project"""
|
||||
LOG.info('reading file contents in %s', proj_obj.name)
|
||||
LOG.debug('reading file contents in %s', proj_obj.name)
|
||||
# Delete any existing files in case the list of files being
|
||||
# managed has changed. This naive, and we can do better, but as a
|
||||
# first version it's OK.
|
||||
_delete_filehandler_data_from_project(session, proj_obj)
|
||||
for file_obj in proj_obj.files:
|
||||
session.delete(file_obj)
|
||||
|
||||
file_handlers = filehandler.load_handlers()
|
||||
|
||||
# FIXME(dhellmann): Concurrency?
|
||||
|
||||
# Now load the files currently being managed by git.
|
||||
|
@ -88,24 +104,25 @@ def _update_project_files(session, proj_obj):
|
|||
session.add(new_file)
|
||||
if any(fnmatch.fnmatch(filename, dnr) for dnr in _DO_NOT_READ):
|
||||
LOG.debug('ignoring contents of %s', fullname)
|
||||
continue
|
||||
with io.open(fullname, mode='r', encoding='utf-8') as f:
|
||||
try:
|
||||
body = f.read()
|
||||
except UnicodeDecodeError:
|
||||
# FIXME(dhellmann): Be smarter about trying other
|
||||
# encodings?
|
||||
LOG.warn('Could not read %s as a UTF-8 encoded file, ignoring',
|
||||
fullname)
|
||||
continue
|
||||
lines = body.splitlines()
|
||||
LOG.debug('%s/%s has %s lines', proj_obj.name, filename, len(lines))
|
||||
for num, content in enumerate(lines, 1):
|
||||
session.add(Line(file=new_file, number=num, content=content))
|
||||
else:
|
||||
with io.open(fullname, mode='r', encoding='utf-8') as f:
|
||||
try:
|
||||
body = f.read()
|
||||
except UnicodeDecodeError:
|
||||
# FIXME(dhellmann): Be smarter about trying other
|
||||
# encodings?
|
||||
LOG.warn('Could not read %s as a UTF-8 encoded file, ignoring',
|
||||
fullname)
|
||||
continue
|
||||
lines = body.splitlines()
|
||||
LOG.debug('%s/%s has %s lines', proj_obj.name, filename, len(lines))
|
||||
for num, content in enumerate(lines, 1):
|
||||
session.add(Line(file=new_file, number=num, content=content))
|
||||
|
||||
# NOTE(dhellmann): Use stevedore to invoke plugins based on
|
||||
# fnmatch of filename being read (use the filename, not the
|
||||
# fullname.
|
||||
# Invoke plugins for processing files in special ways
|
||||
for fh in file_handlers:
|
||||
if fh.obj.supports_file(new_file):
|
||||
fh.obj.process_file(session, new_file)
|
||||
|
||||
|
||||
def discover(repo_root):
|
||||
|
|
|
@ -0,0 +1,59 @@
|
|||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts
|
||||
script_location = alembic
|
||||
|
||||
# template used to generate migration files
|
||||
# file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
# max length of characters to apply to the
|
||||
# "slug" field
|
||||
#truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||
|
||||
|
||||
# Logging configuration
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
|
@ -0,0 +1 @@
|
|||
Generic single-database configuration.
|
|
@ -0,0 +1,72 @@
|
|||
from __future__ import with_statement
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
from logging.config import fileConfig
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
#fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
# from myapp import mymodel
|
||||
# target_metadata = mymodel.Base.metadata
|
||||
target_metadata = None
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline():
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(url=url, target_metadata=target_metadata)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online():
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
"""
|
||||
engine = engine_from_config(
|
||||
config.get_section(config.config_ini_section),
|
||||
prefix='sqlalchemy.',
|
||||
poolclass=pool.NullPool)
|
||||
|
||||
connection = engine.connect()
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata
|
||||
)
|
||||
|
||||
try:
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
|
@ -0,0 +1,22 @@
|
|||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = ${repr(up_revision)}
|
||||
down_revision = ${repr(down_revision)}
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
def upgrade():
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade():
|
||||
${downgrades if downgrades else "pass"}
|
|
@ -0,0 +1,30 @@
|
|||
"""add requirements table
|
||||
|
||||
Revision ID: 203851643975
|
||||
Revises: None
|
||||
Create Date: 2014-11-04 14:02:12.847385
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '203851643975'
|
||||
down_revision = None
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
'requirement',
|
||||
sa.Column('id', sa.Integer, primary_key=True),
|
||||
sa.Column('line_id', sa.Integer,
|
||||
sa.ForeignKey('line.id', name='fk_requirement_line_id')),
|
||||
sa.Column('project_id', sa.Integer,
|
||||
sa.ForeignKey('project.id', name='fk_requirement_project_id')),
|
||||
sa.Column('name', sa.String()),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table('requirement')
|
|
@ -0,0 +1,55 @@
|
|||
import logging
|
||||
import os
|
||||
|
||||
from aeromancer.db import models
|
||||
from aeromancer.requirements import models as req_models
|
||||
from aeromancer import project
|
||||
from aeromancer import utils
|
||||
|
||||
from cliff.lister import Lister
|
||||
|
||||
|
||||
class List(Lister):
|
||||
"""List the requirements for a project"""
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
def get_parser(self, prog_name):
|
||||
parser = super(List, self).get_parser(prog_name)
|
||||
parser.add_argument(
|
||||
'project',
|
||||
help=('project directory name under the project root, '
|
||||
'for example: "stackforge/aeromancer"'),
|
||||
)
|
||||
return parser
|
||||
|
||||
def take_action(self, parsed_args):
|
||||
session = self.app.get_db_session()
|
||||
query = session.query(models.Project).filter(
|
||||
models.Project.name == parsed_args.project
|
||||
)
|
||||
proj_obj = query.one()
|
||||
return (('Name', 'Spec'),
|
||||
((r.name, r.line.content.strip()) for r in proj_obj.requirements))
|
||||
|
||||
|
||||
class Uses(Lister):
|
||||
"""List the projects that use requirement"""
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
def get_parser(self, prog_name):
|
||||
parser = super(Uses, self).get_parser(prog_name)
|
||||
parser.add_argument(
|
||||
'requirement',
|
||||
help='the dist name for the requirement',
|
||||
)
|
||||
return parser
|
||||
|
||||
def take_action(self, parsed_args):
|
||||
session = self.app.get_db_session()
|
||||
query = session.query(req_models.Requirement).filter(
|
||||
req_models.Requirement.name == parsed_args.requirement
|
||||
)
|
||||
return (('Name', 'Spec', 'File'),
|
||||
((r.project.name, r.line.content.strip(), r.line.file.name) for r in query.all()))
|
|
@ -0,0 +1,44 @@
|
|||
import logging
|
||||
|
||||
import pkg_resources
|
||||
|
||||
from aeromancer.filehandler import base
|
||||
from . import models
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RequirementsHandler(base.FileHandler):
|
||||
|
||||
INTERESTING_PATTERNS = ['*requirements*.txt']
|
||||
|
||||
def process_file(self, session, file_obj):
|
||||
LOG.info('loading requirements from %s', file_obj.path)
|
||||
parent_project = file_obj.project
|
||||
for line in file_obj.lines:
|
||||
text = line.content.strip()
|
||||
if not text or text.startswith('#'):
|
||||
continue
|
||||
try:
|
||||
dist_name = pkg_resources.Requirement.parse(text).project_name
|
||||
except ValueError:
|
||||
LOG.warn('could not parse dist name from %r',
|
||||
line.content)
|
||||
continue
|
||||
LOG.info('requirement: %s', dist_name)
|
||||
new_r = models.Requirement(
|
||||
name=dist_name,
|
||||
line=line,
|
||||
project=parent_project,
|
||||
)
|
||||
session.add(new_r)
|
||||
|
||||
def delete_data_for_file(self, session, file_obj):
|
||||
LOG.debug('deleting requirements from %r', file_obj.path)
|
||||
for line in file_obj.lines:
|
||||
query = session.query(models.Requirement).filter(
|
||||
models.Requirement.line_id == line.id
|
||||
)
|
||||
for req in query.all():
|
||||
session.delete(req)
|
||||
return
|
|
@ -0,0 +1,21 @@
|
|||
from sqlalchemy import Column, Integer, String, ForeignKey
|
||||
from sqlalchemy.orm import relationship, backref
|
||||
|
||||
from aeromancer.db import models
|
||||
|
||||
|
||||
class Requirement(models.Base):
|
||||
__tablename__ = 'requirement'
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String, nullable=False)
|
||||
line_id = Column(Integer, ForeignKey('line.id'))
|
||||
line = relationship(
|
||||
models.Line,
|
||||
uselist=False,
|
||||
single_parent=True,
|
||||
)
|
||||
project_id = Column(Integer, ForeignKey('project.id'))
|
||||
project = relationship(
|
||||
models.Project,
|
||||
backref='requirements',
|
||||
)
|
|
@ -55,3 +55,8 @@ aeromancer.cli =
|
|||
remove = aeromancer.cli.project:Remove
|
||||
rescan = aeromancer.cli.project:Rescan
|
||||
discover = aeromancer.cli.project:Discover
|
||||
requirements list = aeromancer.requirements.cli:List
|
||||
what uses = aeromancer.requirements.cli:Uses
|
||||
|
||||
aeromancer.filehandler =
|
||||
requirements = aeromancer.requirements.handler:RequirementsHandler
|
||||
|
|
Loading…
Reference in New Issue