add requirements handling

Add support for file-specific plugins, including schema changes and
models.

Implement scanner for requirements list files and commands to list
requirements for a project and find projects using a specific
requirement.
This commit is contained in:
Doug Hellmann 2014-11-06 15:31:25 +01:00
parent 324a11f69f
commit 581755045a
14 changed files with 426 additions and 32 deletions

View File

@ -1,24 +1,57 @@
import logging
from alembic.config import Config
from alembic import config
from alembic import command
from alembic.environment import EnvironmentContext
from alembic.script import ScriptDirectory
from alembic import environment
from alembic import script
from sqlalchemy import engine_from_config, pool
from aeromancer.db import connect
from aeromancer import filehandler
LOG = logging.getLogger(__name__)
def run_migrations():
config = Config()
config.set_main_option("script_location", "aeromancer.db:alembic")
def _run_migrations_in_location(location):
LOG.debug('loading migrations from %s', location)
url = connect.get_url()
config.set_main_option("sqlalchemy.url", url)
command.upgrade(config, 'head')
# NOTE(dhellmann): Load migration settings from the plugins for
# processing special types of files, and run them.
# We need a unique version_table for each set of migrations.
version_table = location.replace('.', '_') + '_versions'
# Modified version of alembic.command.upgrade().
# command.upgrade(cfg, 'head')
revision = 'head'
cfg = config.Config()
cfg.set_main_option('script_location', location + ':alembic')
cfg.set_main_option("sqlalchemy.url", url)
script_dir = script.ScriptDirectory.from_config(cfg)
def upgrade(rev, context):
return script_dir._upgrade_revs(revision, rev)
with environment.EnvironmentContext(
cfg,
script_dir,
fn=upgrade,
as_sql=False,
starting_rev=None,
destination_rev=revision,
tag=None,
version_table=version_table,
):
script_dir.run_env()
def run_migrations():
_run_migrations_in_location("aeromancer.db")
file_handlers = filehandler.load_handlers()
for fh in file_handlers:
_run_migrations_in_location(fh.entry_point.module_name)
if __name__ == '__main__':

View File

@ -0,0 +1,13 @@
from stevedore import extension
def _load_error_handler(*args, **kwds):
raise
def load_handlers():
return extension.ExtensionManager(
'aeromancer.filehandler',
invoke_on_load=True,
on_load_failure_callback=_load_error_handler,
)

View File

@ -0,0 +1,22 @@
import abc
import fnmatch
class FileHandler(object):
__metaclass__ = abc.ABCMeta
INTERESTING_PATTERNS = []
def supports_file(self, file_obj):
"""Does this plugin want to process the file?
"""
return any(fnmatch.fnmatch(file_obj.path, ip)
for ip in self.INTERESTING_PATTERNS)
@abc.abstractmethod
def process_file(self, session, file_obj):
return
@abc.abstractmethod
def delete_data_for_file(self, session, file_obj):
return

View File

@ -6,15 +6,28 @@ import logging
import os
import subprocess
from aeromancer.db.models import Project, File, Line
from aeromancer import utils
from sqlalchemy.orm.exc import NoResultFound
from aeromancer.db.models import Project, File, Line
from aeromancer import filehandler
from aeromancer import utils
LOG = logging.getLogger(__name__)
def _delete_filehandler_data_from_project(session, proj_obj):
# We have to explicitly have the handlers delete their data
# because the parent-child relationship of the tables is reversed
# because the plugins define the relationships.
file_handlers = filehandler.load_handlers()
LOG.debug('deleting plugin data for %s', proj_obj.name)
for file_obj in proj_obj.files:
for fh in file_handlers:
if fh.obj.supports_file(file_obj):
fh.obj.delete_data_for_file(session, file_obj)
def add_or_update(session, name, path):
"""Create a new project definition or update an existing one"""
query = session.query(Project).filter(Project.name == name)
@ -26,7 +39,6 @@ def add_or_update(session, name, path):
proj_obj = Project(name=name, path=path)
LOG.info('adding project %s from %s', name, path)
session.add(proj_obj)
update(session, proj_obj)
return proj_obj
@ -39,6 +51,7 @@ def remove(session, name):
LOG.info('removing project %s', name)
except NoResultFound:
return
_delete_filehandler_data_from_project(session, proj_obj)
session.delete(proj_obj)
@ -70,13 +83,16 @@ _DO_NOT_READ = [
def _update_project_files(session, proj_obj):
"""Update the files stored for each project"""
LOG.info('reading file contents in %s', proj_obj.name)
LOG.debug('reading file contents in %s', proj_obj.name)
# Delete any existing files in case the list of files being
# managed has changed. This naive, and we can do better, but as a
# first version it's OK.
_delete_filehandler_data_from_project(session, proj_obj)
for file_obj in proj_obj.files:
session.delete(file_obj)
file_handlers = filehandler.load_handlers()
# FIXME(dhellmann): Concurrency?
# Now load the files currently being managed by git.
@ -88,24 +104,25 @@ def _update_project_files(session, proj_obj):
session.add(new_file)
if any(fnmatch.fnmatch(filename, dnr) for dnr in _DO_NOT_READ):
LOG.debug('ignoring contents of %s', fullname)
continue
with io.open(fullname, mode='r', encoding='utf-8') as f:
try:
body = f.read()
except UnicodeDecodeError:
# FIXME(dhellmann): Be smarter about trying other
# encodings?
LOG.warn('Could not read %s as a UTF-8 encoded file, ignoring',
fullname)
continue
lines = body.splitlines()
LOG.debug('%s/%s has %s lines', proj_obj.name, filename, len(lines))
for num, content in enumerate(lines, 1):
session.add(Line(file=new_file, number=num, content=content))
else:
with io.open(fullname, mode='r', encoding='utf-8') as f:
try:
body = f.read()
except UnicodeDecodeError:
# FIXME(dhellmann): Be smarter about trying other
# encodings?
LOG.warn('Could not read %s as a UTF-8 encoded file, ignoring',
fullname)
continue
lines = body.splitlines()
LOG.debug('%s/%s has %s lines', proj_obj.name, filename, len(lines))
for num, content in enumerate(lines, 1):
session.add(Line(file=new_file, number=num, content=content))
# NOTE(dhellmann): Use stevedore to invoke plugins based on
# fnmatch of filename being read (use the filename, not the
# fullname.
# Invoke plugins for processing files in special ways
for fh in file_handlers:
if fh.obj.supports_file(new_file):
fh.obj.process_file(session, new_file)
def discover(repo_root):

View File

View File

@ -0,0 +1,59 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = alembic
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# max length of characters to apply to the
# "slug" field
#truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# sqlalchemy.url = driver://user:pass@localhost/dbname
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

View File

@ -0,0 +1 @@
Generic single-database configuration.

View File

@ -0,0 +1,72 @@
from __future__ import with_statement
from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
#fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(url=url, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
engine = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

View File

@ -0,0 +1,22 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision}
Create Date: ${create_date}
"""
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View File

@ -0,0 +1,30 @@
"""add requirements table
Revision ID: 203851643975
Revises: None
Create Date: 2014-11-04 14:02:12.847385
"""
# revision identifiers, used by Alembic.
revision = '203851643975'
down_revision = None
from alembic import op
import sqlalchemy as sa
def upgrade():
op.create_table(
'requirement',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('line_id', sa.Integer,
sa.ForeignKey('line.id', name='fk_requirement_line_id')),
sa.Column('project_id', sa.Integer,
sa.ForeignKey('project.id', name='fk_requirement_project_id')),
sa.Column('name', sa.String()),
)
def downgrade():
op.drop_table('requirement')

View File

@ -0,0 +1,55 @@
import logging
import os
from aeromancer.db import models
from aeromancer.requirements import models as req_models
from aeromancer import project
from aeromancer import utils
from cliff.lister import Lister
class List(Lister):
"""List the requirements for a project"""
log = logging.getLogger(__name__)
def get_parser(self, prog_name):
parser = super(List, self).get_parser(prog_name)
parser.add_argument(
'project',
help=('project directory name under the project root, '
'for example: "stackforge/aeromancer"'),
)
return parser
def take_action(self, parsed_args):
session = self.app.get_db_session()
query = session.query(models.Project).filter(
models.Project.name == parsed_args.project
)
proj_obj = query.one()
return (('Name', 'Spec'),
((r.name, r.line.content.strip()) for r in proj_obj.requirements))
class Uses(Lister):
"""List the projects that use requirement"""
log = logging.getLogger(__name__)
def get_parser(self, prog_name):
parser = super(Uses, self).get_parser(prog_name)
parser.add_argument(
'requirement',
help='the dist name for the requirement',
)
return parser
def take_action(self, parsed_args):
session = self.app.get_db_session()
query = session.query(req_models.Requirement).filter(
req_models.Requirement.name == parsed_args.requirement
)
return (('Name', 'Spec', 'File'),
((r.project.name, r.line.content.strip(), r.line.file.name) for r in query.all()))

View File

@ -0,0 +1,44 @@
import logging
import pkg_resources
from aeromancer.filehandler import base
from . import models
LOG = logging.getLogger(__name__)
class RequirementsHandler(base.FileHandler):
INTERESTING_PATTERNS = ['*requirements*.txt']
def process_file(self, session, file_obj):
LOG.info('loading requirements from %s', file_obj.path)
parent_project = file_obj.project
for line in file_obj.lines:
text = line.content.strip()
if not text or text.startswith('#'):
continue
try:
dist_name = pkg_resources.Requirement.parse(text).project_name
except ValueError:
LOG.warn('could not parse dist name from %r',
line.content)
continue
LOG.info('requirement: %s', dist_name)
new_r = models.Requirement(
name=dist_name,
line=line,
project=parent_project,
)
session.add(new_r)
def delete_data_for_file(self, session, file_obj):
LOG.debug('deleting requirements from %r', file_obj.path)
for line in file_obj.lines:
query = session.query(models.Requirement).filter(
models.Requirement.line_id == line.id
)
for req in query.all():
session.delete(req)
return

View File

@ -0,0 +1,21 @@
from sqlalchemy import Column, Integer, String, ForeignKey
from sqlalchemy.orm import relationship, backref
from aeromancer.db import models
class Requirement(models.Base):
__tablename__ = 'requirement'
id = Column(Integer, primary_key=True)
name = Column(String, nullable=False)
line_id = Column(Integer, ForeignKey('line.id'))
line = relationship(
models.Line,
uselist=False,
single_parent=True,
)
project_id = Column(Integer, ForeignKey('project.id'))
project = relationship(
models.Project,
backref='requirements',
)

View File

@ -55,3 +55,8 @@ aeromancer.cli =
remove = aeromancer.cli.project:Remove
rescan = aeromancer.cli.project:Rescan
discover = aeromancer.cli.project:Discover
requirements list = aeromancer.requirements.cli:List
what uses = aeromancer.requirements.cli:Uses
aeromancer.filehandler =
requirements = aeromancer.requirements.handler:RequirementsHandler