add requirements handling
Add support for file-specific plugins, including schema changes and models. Implement scanner for requirements list files and commands to list requirements for a project and find projects using a specific requirement.
This commit is contained in:
parent
324a11f69f
commit
581755045a
|
@ -1,24 +1,57 @@
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from alembic.config import Config
|
from alembic import config
|
||||||
from alembic import command
|
from alembic import command
|
||||||
from alembic.environment import EnvironmentContext
|
from alembic import environment
|
||||||
from alembic.script import ScriptDirectory
|
from alembic import script
|
||||||
|
|
||||||
from sqlalchemy import engine_from_config, pool
|
from sqlalchemy import engine_from_config, pool
|
||||||
|
|
||||||
from aeromancer.db import connect
|
from aeromancer.db import connect
|
||||||
|
from aeromancer import filehandler
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def run_migrations():
|
def _run_migrations_in_location(location):
|
||||||
config = Config()
|
LOG.debug('loading migrations from %s', location)
|
||||||
config.set_main_option("script_location", "aeromancer.db:alembic")
|
|
||||||
url = connect.get_url()
|
url = connect.get_url()
|
||||||
config.set_main_option("sqlalchemy.url", url)
|
|
||||||
command.upgrade(config, 'head')
|
# We need a unique version_table for each set of migrations.
|
||||||
# NOTE(dhellmann): Load migration settings from the plugins for
|
version_table = location.replace('.', '_') + '_versions'
|
||||||
# processing special types of files, and run them.
|
|
||||||
|
# Modified version of alembic.command.upgrade().
|
||||||
|
# command.upgrade(cfg, 'head')
|
||||||
|
revision = 'head'
|
||||||
|
|
||||||
|
cfg = config.Config()
|
||||||
|
cfg.set_main_option('script_location', location + ':alembic')
|
||||||
|
cfg.set_main_option("sqlalchemy.url", url)
|
||||||
|
|
||||||
|
script_dir = script.ScriptDirectory.from_config(cfg)
|
||||||
|
|
||||||
|
def upgrade(rev, context):
|
||||||
|
return script_dir._upgrade_revs(revision, rev)
|
||||||
|
|
||||||
|
with environment.EnvironmentContext(
|
||||||
|
cfg,
|
||||||
|
script_dir,
|
||||||
|
fn=upgrade,
|
||||||
|
as_sql=False,
|
||||||
|
starting_rev=None,
|
||||||
|
destination_rev=revision,
|
||||||
|
tag=None,
|
||||||
|
version_table=version_table,
|
||||||
|
):
|
||||||
|
script_dir.run_env()
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations():
|
||||||
|
_run_migrations_in_location("aeromancer.db")
|
||||||
|
file_handlers = filehandler.load_handlers()
|
||||||
|
for fh in file_handlers:
|
||||||
|
_run_migrations_in_location(fh.entry_point.module_name)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -0,0 +1,13 @@
|
||||||
|
from stevedore import extension
|
||||||
|
|
||||||
|
|
||||||
|
def _load_error_handler(*args, **kwds):
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def load_handlers():
|
||||||
|
return extension.ExtensionManager(
|
||||||
|
'aeromancer.filehandler',
|
||||||
|
invoke_on_load=True,
|
||||||
|
on_load_failure_callback=_load_error_handler,
|
||||||
|
)
|
|
@ -0,0 +1,22 @@
|
||||||
|
import abc
|
||||||
|
import fnmatch
|
||||||
|
|
||||||
|
|
||||||
|
class FileHandler(object):
|
||||||
|
__metaclass__ = abc.ABCMeta
|
||||||
|
|
||||||
|
INTERESTING_PATTERNS = []
|
||||||
|
|
||||||
|
def supports_file(self, file_obj):
|
||||||
|
"""Does this plugin want to process the file?
|
||||||
|
"""
|
||||||
|
return any(fnmatch.fnmatch(file_obj.path, ip)
|
||||||
|
for ip in self.INTERESTING_PATTERNS)
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def process_file(self, session, file_obj):
|
||||||
|
return
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def delete_data_for_file(self, session, file_obj):
|
||||||
|
return
|
|
@ -6,15 +6,28 @@ import logging
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
from aeromancer.db.models import Project, File, Line
|
|
||||||
from aeromancer import utils
|
|
||||||
|
|
||||||
from sqlalchemy.orm.exc import NoResultFound
|
from sqlalchemy.orm.exc import NoResultFound
|
||||||
|
|
||||||
|
from aeromancer.db.models import Project, File, Line
|
||||||
|
from aeromancer import filehandler
|
||||||
|
from aeromancer import utils
|
||||||
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _delete_filehandler_data_from_project(session, proj_obj):
|
||||||
|
# We have to explicitly have the handlers delete their data
|
||||||
|
# because the parent-child relationship of the tables is reversed
|
||||||
|
# because the plugins define the relationships.
|
||||||
|
file_handlers = filehandler.load_handlers()
|
||||||
|
LOG.debug('deleting plugin data for %s', proj_obj.name)
|
||||||
|
for file_obj in proj_obj.files:
|
||||||
|
for fh in file_handlers:
|
||||||
|
if fh.obj.supports_file(file_obj):
|
||||||
|
fh.obj.delete_data_for_file(session, file_obj)
|
||||||
|
|
||||||
|
|
||||||
def add_or_update(session, name, path):
|
def add_or_update(session, name, path):
|
||||||
"""Create a new project definition or update an existing one"""
|
"""Create a new project definition or update an existing one"""
|
||||||
query = session.query(Project).filter(Project.name == name)
|
query = session.query(Project).filter(Project.name == name)
|
||||||
|
@ -26,7 +39,6 @@ def add_or_update(session, name, path):
|
||||||
proj_obj = Project(name=name, path=path)
|
proj_obj = Project(name=name, path=path)
|
||||||
LOG.info('adding project %s from %s', name, path)
|
LOG.info('adding project %s from %s', name, path)
|
||||||
session.add(proj_obj)
|
session.add(proj_obj)
|
||||||
|
|
||||||
update(session, proj_obj)
|
update(session, proj_obj)
|
||||||
return proj_obj
|
return proj_obj
|
||||||
|
|
||||||
|
@ -39,6 +51,7 @@ def remove(session, name):
|
||||||
LOG.info('removing project %s', name)
|
LOG.info('removing project %s', name)
|
||||||
except NoResultFound:
|
except NoResultFound:
|
||||||
return
|
return
|
||||||
|
_delete_filehandler_data_from_project(session, proj_obj)
|
||||||
session.delete(proj_obj)
|
session.delete(proj_obj)
|
||||||
|
|
||||||
|
|
||||||
|
@ -70,13 +83,16 @@ _DO_NOT_READ = [
|
||||||
|
|
||||||
def _update_project_files(session, proj_obj):
|
def _update_project_files(session, proj_obj):
|
||||||
"""Update the files stored for each project"""
|
"""Update the files stored for each project"""
|
||||||
LOG.info('reading file contents in %s', proj_obj.name)
|
LOG.debug('reading file contents in %s', proj_obj.name)
|
||||||
# Delete any existing files in case the list of files being
|
# Delete any existing files in case the list of files being
|
||||||
# managed has changed. This naive, and we can do better, but as a
|
# managed has changed. This naive, and we can do better, but as a
|
||||||
# first version it's OK.
|
# first version it's OK.
|
||||||
|
_delete_filehandler_data_from_project(session, proj_obj)
|
||||||
for file_obj in proj_obj.files:
|
for file_obj in proj_obj.files:
|
||||||
session.delete(file_obj)
|
session.delete(file_obj)
|
||||||
|
|
||||||
|
file_handlers = filehandler.load_handlers()
|
||||||
|
|
||||||
# FIXME(dhellmann): Concurrency?
|
# FIXME(dhellmann): Concurrency?
|
||||||
|
|
||||||
# Now load the files currently being managed by git.
|
# Now load the files currently being managed by git.
|
||||||
|
@ -88,24 +104,25 @@ def _update_project_files(session, proj_obj):
|
||||||
session.add(new_file)
|
session.add(new_file)
|
||||||
if any(fnmatch.fnmatch(filename, dnr) for dnr in _DO_NOT_READ):
|
if any(fnmatch.fnmatch(filename, dnr) for dnr in _DO_NOT_READ):
|
||||||
LOG.debug('ignoring contents of %s', fullname)
|
LOG.debug('ignoring contents of %s', fullname)
|
||||||
continue
|
else:
|
||||||
with io.open(fullname, mode='r', encoding='utf-8') as f:
|
with io.open(fullname, mode='r', encoding='utf-8') as f:
|
||||||
try:
|
try:
|
||||||
body = f.read()
|
body = f.read()
|
||||||
except UnicodeDecodeError:
|
except UnicodeDecodeError:
|
||||||
# FIXME(dhellmann): Be smarter about trying other
|
# FIXME(dhellmann): Be smarter about trying other
|
||||||
# encodings?
|
# encodings?
|
||||||
LOG.warn('Could not read %s as a UTF-8 encoded file, ignoring',
|
LOG.warn('Could not read %s as a UTF-8 encoded file, ignoring',
|
||||||
fullname)
|
fullname)
|
||||||
continue
|
continue
|
||||||
lines = body.splitlines()
|
lines = body.splitlines()
|
||||||
LOG.debug('%s/%s has %s lines', proj_obj.name, filename, len(lines))
|
LOG.debug('%s/%s has %s lines', proj_obj.name, filename, len(lines))
|
||||||
for num, content in enumerate(lines, 1):
|
for num, content in enumerate(lines, 1):
|
||||||
session.add(Line(file=new_file, number=num, content=content))
|
session.add(Line(file=new_file, number=num, content=content))
|
||||||
|
|
||||||
# NOTE(dhellmann): Use stevedore to invoke plugins based on
|
# Invoke plugins for processing files in special ways
|
||||||
# fnmatch of filename being read (use the filename, not the
|
for fh in file_handlers:
|
||||||
# fullname.
|
if fh.obj.supports_file(new_file):
|
||||||
|
fh.obj.process_file(session, new_file)
|
||||||
|
|
||||||
|
|
||||||
def discover(repo_root):
|
def discover(repo_root):
|
||||||
|
|
|
@ -0,0 +1,59 @@
|
||||||
|
# A generic, single database configuration.
|
||||||
|
|
||||||
|
[alembic]
|
||||||
|
# path to migration scripts
|
||||||
|
script_location = alembic
|
||||||
|
|
||||||
|
# template used to generate migration files
|
||||||
|
# file_template = %%(rev)s_%%(slug)s
|
||||||
|
|
||||||
|
# max length of characters to apply to the
|
||||||
|
# "slug" field
|
||||||
|
#truncate_slug_length = 40
|
||||||
|
|
||||||
|
# set to 'true' to run the environment during
|
||||||
|
# the 'revision' command, regardless of autogenerate
|
||||||
|
# revision_environment = false
|
||||||
|
|
||||||
|
# set to 'true' to allow .pyc and .pyo files without
|
||||||
|
# a source .py file to be detected as revisions in the
|
||||||
|
# versions/ directory
|
||||||
|
# sourceless = false
|
||||||
|
|
||||||
|
# sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||||
|
|
||||||
|
|
||||||
|
# Logging configuration
|
||||||
|
[loggers]
|
||||||
|
keys = root,sqlalchemy,alembic
|
||||||
|
|
||||||
|
[handlers]
|
||||||
|
keys = console
|
||||||
|
|
||||||
|
[formatters]
|
||||||
|
keys = generic
|
||||||
|
|
||||||
|
[logger_root]
|
||||||
|
level = WARN
|
||||||
|
handlers = console
|
||||||
|
qualname =
|
||||||
|
|
||||||
|
[logger_sqlalchemy]
|
||||||
|
level = WARN
|
||||||
|
handlers =
|
||||||
|
qualname = sqlalchemy.engine
|
||||||
|
|
||||||
|
[logger_alembic]
|
||||||
|
level = INFO
|
||||||
|
handlers =
|
||||||
|
qualname = alembic
|
||||||
|
|
||||||
|
[handler_console]
|
||||||
|
class = StreamHandler
|
||||||
|
args = (sys.stderr,)
|
||||||
|
level = NOTSET
|
||||||
|
formatter = generic
|
||||||
|
|
||||||
|
[formatter_generic]
|
||||||
|
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||||
|
datefmt = %H:%M:%S
|
|
@ -0,0 +1 @@
|
||||||
|
Generic single-database configuration.
|
|
@ -0,0 +1,72 @@
|
||||||
|
from __future__ import with_statement
|
||||||
|
from alembic import context
|
||||||
|
from sqlalchemy import engine_from_config, pool
|
||||||
|
from logging.config import fileConfig
|
||||||
|
|
||||||
|
# this is the Alembic Config object, which provides
|
||||||
|
# access to the values within the .ini file in use.
|
||||||
|
config = context.config
|
||||||
|
|
||||||
|
# Interpret the config file for Python logging.
|
||||||
|
# This line sets up loggers basically.
|
||||||
|
#fileConfig(config.config_file_name)
|
||||||
|
|
||||||
|
# add your model's MetaData object here
|
||||||
|
# for 'autogenerate' support
|
||||||
|
# from myapp import mymodel
|
||||||
|
# target_metadata = mymodel.Base.metadata
|
||||||
|
target_metadata = None
|
||||||
|
|
||||||
|
# other values from the config, defined by the needs of env.py,
|
||||||
|
# can be acquired:
|
||||||
|
# my_important_option = config.get_main_option("my_important_option")
|
||||||
|
# ... etc.
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_offline():
|
||||||
|
"""Run migrations in 'offline' mode.
|
||||||
|
|
||||||
|
This configures the context with just a URL
|
||||||
|
and not an Engine, though an Engine is acceptable
|
||||||
|
here as well. By skipping the Engine creation
|
||||||
|
we don't even need a DBAPI to be available.
|
||||||
|
|
||||||
|
Calls to context.execute() here emit the given string to the
|
||||||
|
script output.
|
||||||
|
|
||||||
|
"""
|
||||||
|
url = config.get_main_option("sqlalchemy.url")
|
||||||
|
context.configure(url=url, target_metadata=target_metadata)
|
||||||
|
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_online():
|
||||||
|
"""Run migrations in 'online' mode.
|
||||||
|
|
||||||
|
In this scenario we need to create an Engine
|
||||||
|
and associate a connection with the context.
|
||||||
|
|
||||||
|
"""
|
||||||
|
engine = engine_from_config(
|
||||||
|
config.get_section(config.config_ini_section),
|
||||||
|
prefix='sqlalchemy.',
|
||||||
|
poolclass=pool.NullPool)
|
||||||
|
|
||||||
|
connection = engine.connect()
|
||||||
|
context.configure(
|
||||||
|
connection=connection,
|
||||||
|
target_metadata=target_metadata
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
if context.is_offline_mode():
|
||||||
|
run_migrations_offline()
|
||||||
|
else:
|
||||||
|
run_migrations_online()
|
|
@ -0,0 +1,22 @@
|
||||||
|
"""${message}
|
||||||
|
|
||||||
|
Revision ID: ${up_revision}
|
||||||
|
Revises: ${down_revision}
|
||||||
|
Create Date: ${create_date}
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = ${repr(up_revision)}
|
||||||
|
down_revision = ${repr(down_revision)}
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
${imports if imports else ""}
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
${upgrades if upgrades else "pass"}
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
${downgrades if downgrades else "pass"}
|
|
@ -0,0 +1,30 @@
|
||||||
|
"""add requirements table
|
||||||
|
|
||||||
|
Revision ID: 203851643975
|
||||||
|
Revises: None
|
||||||
|
Create Date: 2014-11-04 14:02:12.847385
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = '203851643975'
|
||||||
|
down_revision = None
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.create_table(
|
||||||
|
'requirement',
|
||||||
|
sa.Column('id', sa.Integer, primary_key=True),
|
||||||
|
sa.Column('line_id', sa.Integer,
|
||||||
|
sa.ForeignKey('line.id', name='fk_requirement_line_id')),
|
||||||
|
sa.Column('project_id', sa.Integer,
|
||||||
|
sa.ForeignKey('project.id', name='fk_requirement_project_id')),
|
||||||
|
sa.Column('name', sa.String()),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.drop_table('requirement')
|
|
@ -0,0 +1,55 @@
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
from aeromancer.db import models
|
||||||
|
from aeromancer.requirements import models as req_models
|
||||||
|
from aeromancer import project
|
||||||
|
from aeromancer import utils
|
||||||
|
|
||||||
|
from cliff.lister import Lister
|
||||||
|
|
||||||
|
|
||||||
|
class List(Lister):
|
||||||
|
"""List the requirements for a project"""
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def get_parser(self, prog_name):
|
||||||
|
parser = super(List, self).get_parser(prog_name)
|
||||||
|
parser.add_argument(
|
||||||
|
'project',
|
||||||
|
help=('project directory name under the project root, '
|
||||||
|
'for example: "stackforge/aeromancer"'),
|
||||||
|
)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
def take_action(self, parsed_args):
|
||||||
|
session = self.app.get_db_session()
|
||||||
|
query = session.query(models.Project).filter(
|
||||||
|
models.Project.name == parsed_args.project
|
||||||
|
)
|
||||||
|
proj_obj = query.one()
|
||||||
|
return (('Name', 'Spec'),
|
||||||
|
((r.name, r.line.content.strip()) for r in proj_obj.requirements))
|
||||||
|
|
||||||
|
|
||||||
|
class Uses(Lister):
|
||||||
|
"""List the projects that use requirement"""
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def get_parser(self, prog_name):
|
||||||
|
parser = super(Uses, self).get_parser(prog_name)
|
||||||
|
parser.add_argument(
|
||||||
|
'requirement',
|
||||||
|
help='the dist name for the requirement',
|
||||||
|
)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
def take_action(self, parsed_args):
|
||||||
|
session = self.app.get_db_session()
|
||||||
|
query = session.query(req_models.Requirement).filter(
|
||||||
|
req_models.Requirement.name == parsed_args.requirement
|
||||||
|
)
|
||||||
|
return (('Name', 'Spec', 'File'),
|
||||||
|
((r.project.name, r.line.content.strip(), r.line.file.name) for r in query.all()))
|
|
@ -0,0 +1,44 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import pkg_resources
|
||||||
|
|
||||||
|
from aeromancer.filehandler import base
|
||||||
|
from . import models
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class RequirementsHandler(base.FileHandler):
|
||||||
|
|
||||||
|
INTERESTING_PATTERNS = ['*requirements*.txt']
|
||||||
|
|
||||||
|
def process_file(self, session, file_obj):
|
||||||
|
LOG.info('loading requirements from %s', file_obj.path)
|
||||||
|
parent_project = file_obj.project
|
||||||
|
for line in file_obj.lines:
|
||||||
|
text = line.content.strip()
|
||||||
|
if not text or text.startswith('#'):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
dist_name = pkg_resources.Requirement.parse(text).project_name
|
||||||
|
except ValueError:
|
||||||
|
LOG.warn('could not parse dist name from %r',
|
||||||
|
line.content)
|
||||||
|
continue
|
||||||
|
LOG.info('requirement: %s', dist_name)
|
||||||
|
new_r = models.Requirement(
|
||||||
|
name=dist_name,
|
||||||
|
line=line,
|
||||||
|
project=parent_project,
|
||||||
|
)
|
||||||
|
session.add(new_r)
|
||||||
|
|
||||||
|
def delete_data_for_file(self, session, file_obj):
|
||||||
|
LOG.debug('deleting requirements from %r', file_obj.path)
|
||||||
|
for line in file_obj.lines:
|
||||||
|
query = session.query(models.Requirement).filter(
|
||||||
|
models.Requirement.line_id == line.id
|
||||||
|
)
|
||||||
|
for req in query.all():
|
||||||
|
session.delete(req)
|
||||||
|
return
|
|
@ -0,0 +1,21 @@
|
||||||
|
from sqlalchemy import Column, Integer, String, ForeignKey
|
||||||
|
from sqlalchemy.orm import relationship, backref
|
||||||
|
|
||||||
|
from aeromancer.db import models
|
||||||
|
|
||||||
|
|
||||||
|
class Requirement(models.Base):
|
||||||
|
__tablename__ = 'requirement'
|
||||||
|
id = Column(Integer, primary_key=True)
|
||||||
|
name = Column(String, nullable=False)
|
||||||
|
line_id = Column(Integer, ForeignKey('line.id'))
|
||||||
|
line = relationship(
|
||||||
|
models.Line,
|
||||||
|
uselist=False,
|
||||||
|
single_parent=True,
|
||||||
|
)
|
||||||
|
project_id = Column(Integer, ForeignKey('project.id'))
|
||||||
|
project = relationship(
|
||||||
|
models.Project,
|
||||||
|
backref='requirements',
|
||||||
|
)
|
|
@ -55,3 +55,8 @@ aeromancer.cli =
|
||||||
remove = aeromancer.cli.project:Remove
|
remove = aeromancer.cli.project:Remove
|
||||||
rescan = aeromancer.cli.project:Rescan
|
rescan = aeromancer.cli.project:Rescan
|
||||||
discover = aeromancer.cli.project:Discover
|
discover = aeromancer.cli.project:Discover
|
||||||
|
requirements list = aeromancer.requirements.cli:List
|
||||||
|
what uses = aeromancer.requirements.cli:Uses
|
||||||
|
|
||||||
|
aeromancer.filehandler =
|
||||||
|
requirements = aeromancer.requirements.handler:RequirementsHandler
|
||||||
|
|
Loading…
Reference in New Issue