Add migration and shell

This commit is contained in:
Masayuki Igawa 2016-04-19 19:00:55 +09:00
parent 5a4708f8cc
commit 2d3cbedc89
13 changed files with 548 additions and 1 deletions

2
.gitignore vendored
View File

@ -1,3 +1,5 @@
alembic.ini
*.py[cod]
# C extensions

View File

70
coverage2sql/db/api.py Normal file
View File

@ -0,0 +1,70 @@
# Copyright 2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import datetime
from oslo_config import cfg
#from oslo_db.sqlalchemy import session as db_session
import six
import sqlalchemy
from sqlalchemy.engine.url import make_url
import logging
from coverage2sql.db import models
#from coverage2sql import exceptions
#from coverage2sql import read_coverage
CONF = cfg.CONF
CONF.register_cli_opt(cfg.BoolOpt('verbose', short='v', default=False,
help='Verbose output including logging of '
'SQL statements'))
DAY_SECONDS = 60 * 60 * 24
_facades = {}
def _create_facade_lazily():
global _facades
db_url = make_url(CONF.database.connection)
db_backend = db_url.get_backend_name()
facade = _facades.get(db_backend)
if facade is None:
facade = db_session.EngineFacade(
CONF.database.connection,
**dict(six.iteritems(CONF.database)))
_facades[db_backend] = facade
return facade
def get_session(autocommit=True, expire_on_commit=False):
"""Get a new sqlalchemy Session instance
:param bool autocommit: Enable autocommit mode for the session.
:param bool expire_on_commit: Expire the session on commit defaults False.
"""
facade = _create_facade_lazily()
session = facade.get_session(autocommit=autocommit,
expire_on_commit=expire_on_commit)
# if --verbose was specified, turn on SQL logging
# note that this is done after the session has been initialized so that
# we can override the default sqlalchemy logging
if CONF.get('verbose', False):
logging.basicConfig()
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
return session

58
coverage2sql/db/models.py Normal file
View File

@ -0,0 +1,58 @@
# Copyright 2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import uuid
#from oslo_db.sqlalchemy import models # noqa
import six
import sqlalchemy as sa
from sqlalchemy.ext import declarative
BASE = declarative.declarative_base()
class CoverageBase(object):
"""Base class for Coverage Models."""
__table_args__ = {'mysql_engine': 'InnoDB'}
__table_initialized__ = False
def save(self, session=None):
from coverage2sql.db import api as db_api
super(CoverageBase, self).save(session or db_api.get_session())
def keys(self):
return list(self.__dict__.keys())
def values(self):
return self.__dict__.values()
def items(self):
return self.__dict__.items()
def to_dict(self):
d = self.__dict__.copy()
d.pop("_sa_instance_state")
return d
class Coverage(BASE, CoverageBase):
__tablename__ = 'coverages'
__table_args__ = (sa.Index('ix_project_name', 'project_name'), )
id = sa.Column(sa.BigInteger, primary_key=True)
project_name = sa.Column(sa.String(256),
nullable=False)
coverage_rate = sa.Column(sa.Float())
report_time = sa.Column(sa.DateTime())
report_time_microsecond = sa.Column(sa.Integer(), default=0)

View File

@ -0,0 +1 @@
Generic single-database configuration.

View File

@ -0,0 +1,186 @@
# Copyright 2012 New Dream Network, LLC (DreamHost)
# Copyright 2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import sys
from alembic import command as alembic_command
from alembic import config as alembic_config
from alembic import script as alembic_script
from alembic import util as alembic_util
from oslo_config import cfg
#from oslo_db import options
from coverage2sql.db import api as db_api
HEAD_FILENAME = 'HEAD'
def state_path_def(*args):
"""Return an uninterpolated path relative to $state_path."""
return os.path.join('$state_path', *args)
MIGRATION_OPTS = [
cfg.BoolOpt('disable-microsecond-data-migration', short='d', default=False,
help="If set to true this option will skip the data migration"
" part of the microsecond migration. The schema changes "
"will still be run. If the database has already stripped "
"out the microseconds from the timestamps this will skip "
"converting the microsecond field from the timestamps "
"into a separate column"),
]
CONF = cfg.CONF
#CONF.register_cli_opts(options.database_opts, group='database')
CONF.register_cli_opts(MIGRATION_OPTS)
CONF.import_opt('verbose', 'coverage2sql.db.api')
def do_alembic_command(config, cmd, *args, **kwargs):
try:
getattr(alembic_command, cmd)(config, *args, **kwargs)
except alembic_util.CommandError as e:
alembic_util.err(str(e))
def do_check_migration(config, cmd):
do_alembic_command(config, 'branches')
validate_head_file(config)
def do_upgrade_downgrade(config, cmd):
if not CONF.command.revision and not CONF.command.delta:
raise SystemExit('You must provide a revision or relative delta')
revision = CONF.command.revision
if CONF.command.delta:
sign = '+' if CONF.command.name == 'upgrade' else '-'
revision = sign + str(CONF.command.delta)
else:
revision = CONF.command.revision
do_alembic_command(config, cmd, revision, sql=CONF.command.sql)
def do_stamp(config, cmd):
do_alembic_command(config, cmd,
CONF.command.revision,
sql=CONF.command.sql)
def do_revision(config, cmd):
do_alembic_command(config, cmd,
message=CONF.command.message,
autogenerate=CONF.command.autogenerate,
sql=CONF.command.sql)
update_head_file(config)
def validate_head_file(config):
script = alembic_script.ScriptDirectory.from_config(config)
if len(script.get_heads()) > 1:
alembic_util.err('Timeline branches unable to generate timeline')
head_path = os.path.join(script.versions, HEAD_FILENAME)
if (os.path.isfile(head_path) and
open(head_path).read().strip() == script.get_current_head()):
return
else:
alembic_util.err('HEAD file does not match migration timeline head')
def expire_old(config, cmd):
expire_age = int(CONF.command.expire_age)
if not CONF.command.no_runs:
print('Expiring old runs.')
db_api.delete_old_runs(expire_age)
if not CONF.command.no_test_runs:
print('Expiring old test_runs')
db_api.delete_old_test_runs(expire_age)
def update_head_file(config):
script = alembic_script.ScriptDirectory.from_config(config)
if len(script.get_heads()) > 1:
alembic_util.err('Timeline branches unable to generate timeline')
head_path = os.path.join(script.versions, HEAD_FILENAME)
with open(head_path, 'w+') as f:
f.write(script.get_current_head())
def add_command_parsers(subparsers):
for name in ['current', 'history', 'branches']:
parser = subparsers.add_parser(name)
parser.set_defaults(func=do_alembic_command)
parser = subparsers.add_parser('check_migration')
parser.set_defaults(func=do_check_migration)
for name in ['upgrade', 'downgrade']:
parser = subparsers.add_parser(name)
parser.add_argument('--delta', type=int)
parser.add_argument('--sql', action='store_true')
parser.add_argument('revision', nargs='?')
parser.add_argument('--mysql-engine',
default='',
help='Change MySQL storage engine of current '
'existing tables')
parser.set_defaults(func=do_upgrade_downgrade)
parser = subparsers.add_parser('stamp')
parser.add_argument('--sql', action='store_true')
parser.add_argument('revision')
parser.set_defaults(func=do_stamp)
parser = subparsers.add_parser('revision')
parser.add_argument('-m', '--message')
parser.add_argument('--autogenerate', action='store_true')
parser.add_argument('--sql', action='store_true')
parser.set_defaults(func=do_revision)
parser = subparsers.add_parser('expire',
help="delete old rows from runs and "
"test_runs tables")
parser.add_argument('--no-runs', action='store_true',
help="Don't delete any rows in the runs table")
parser.add_argument('--no-test-runs', action='store_true',
help="Don't delete any rows in the test_runs table")
parser.add_argument('--expire-age', '-e', default=186,
help="Number of days into the past to use as the "
"expiration point")
parser.set_defaults(func=expire_old)
command_opt = cfg.SubCommandOpt('command',
title='Command',
help='Available commands',
handler=add_command_parsers)
CONF.register_cli_opt(command_opt)
def main():
config = alembic_config.Config(os.path.join(os.path.dirname(__file__),
'alembic.ini'))
config.set_main_option('script_location',
'coverage2sql:migrations')
config.coverage2sql_config = CONF
CONF()
CONF.command.func(config, CONF.command.name)
if __name__ == "__main__":
sys.exit(main())

View File

@ -0,0 +1,85 @@
# Copyright 2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import with_statement
from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url, target_metadata=target_metadata, literal_binds=True)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

View File

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View File

@ -0,0 +1,31 @@
"""Add coverages table
Revision ID: 52dfb338f74e
Revises:
Create Date: 2016-04-19 18:16:52.780046
"""
# revision identifiers, used by Alembic.
revision = '52dfb338f74e'
down_revision = None
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
def upgrade():
op.create_table('coverages',
sa.Column('id', sa.BigInteger(), primary_key=True),
sa.Column('project_name', sa.String(256), nullable=False),
sa.Column('coverage_rate', sa.Float()),
sa.Column('report_time', sa.DateTime()),
sa.Column('report_time_microsecond', sa.Integer(),
default=0),
mysql_engine='InnoDB')
def downgrade():
pass

76
coverage2sql/shell.py Normal file
View File

@ -0,0 +1,76 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import sys
from oslo_config import cfg
# from oslo_db import options
from pbr import version
from stevedore import enabled
from coverage2sql.db import api
# from coverage2sql import exceptions
# from coverage2sql import read_subunit as subunit
CONF = cfg.CONF
CONF.import_opt('verbose', 'coverage2sql.db.api')
SHELL_OPTS = [
cfg.MultiStrOpt('coverage_files', positional=True,
help='list of coverage files to put into the database'),
]
_version_ = version.VersionInfo('coverage2sql').version_string()
def cli_opts():
for opt in SHELL_OPTS:
CONF.register_cli_opt(opt)
def list_opts():
"""Return a list of oslo.config options available.
The purpose of this is to allow tools like the Oslo sample config file
generator to discover the options exposed to users.
"""
return [('DEFAULT', copy.deepcopy(SHELL_OPTS))]
def parse_args(argv, default_config_files=None):
# cfg.CONF.register_cli_opts(options.database_opts, group='database')
cfg.CONF(argv[1:], project='coverage2sql', version=_version_,
default_config_files=default_config_files)
def process_results(results):
print(results)
def main():
cli_opts()
parse_args(sys.argv)
if CONF.coverage_files:
print("From file:")
process_results("FIXME") # FIXME
else:
print("From stdin:")
process_results("FIXME") # FIXME
if __name__ == "__main__":
sys.exit(main())

View File

@ -3,3 +3,6 @@
# process, which may cause wedges in the gate later.
pbr>=1.6
SQLAlchemy>=0.8.2
alembic>=0.4.1
oslo.config>=1.4.0.0a3

View File

@ -16,13 +16,23 @@ classifier =
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.3
Programming Language :: Python :: 3.4
[files]
packages =
coverage2sql
[global]
setup-hooks =
pbr.hooks.setup_hook
[entry_points]
console_scripts =
coverage2sql = coverage2sql.shell:main
coverage2sql-db-manage = coverage2sql.migrations.cli:main
oslo.config.opts =
coverage2sql.shell = coverage2sql.shell:list_opts
[build_sphinx]
source-dir = doc/source
build-dir = doc/build

View File

@ -8,6 +8,7 @@ coverage>=3.6
python-subunit>=0.0.18
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
oslosphinx>=2.5.0 # Apache-2.0
PyMySql
oslotest>=1.10.0 # Apache-2.0
testrepository>=0.0.18
testscenarios>=0.4