From 856a9739dab34e8a9b0172a907497e201daeedb3 Mon Sep 17 00:00:00 2001 From: Masayuki Igawa Date: Tue, 7 Feb 2017 18:22:20 +0900 Subject: [PATCH] Add files table This commit adds 'files' table. This files table has coverage_id column as a foreign key for the coverages table. And this commit also adds a release note for it. Change-Id: I8998a79a1ba79bbdab1cd79810cc85bcbccbe7d8 --- coverage2sql/db/api.py | 33 +++++++++++- coverage2sql/db/models.py | 14 +++++ .../52dfb338f74e_add_coverages_table.py | 2 +- .../versions/79dead6f7c26_add_files_table.py | 54 +++++++++++++++++++ coverage2sql/read_coverage.py | 25 ++++++++- coverage2sql/shell.py | 9 ++-- coverage2sql/tests/db/test_api.py | 11 ++++ coverage2sql/tests/test_shell.py | 4 ++ .../add-files-table-e60ac2e98b5f543b.yaml | 4 ++ releasenotes/source/index.rst | 9 ++-- requirements.txt | 2 + test-requirements.txt | 1 - 12 files changed, 155 insertions(+), 13 deletions(-) create mode 100644 coverage2sql/migrations/versions/79dead6f7c26_add_files_table.py create mode 100644 releasenotes/notes/add-files-table-e60ac2e98b5f543b.yaml diff --git a/coverage2sql/db/api.py b/coverage2sql/db/api.py index 001127e..a425e0a 100644 --- a/coverage2sql/db/api.py +++ b/coverage2sql/db/api.py @@ -70,8 +70,8 @@ def get_session(autocommit=True, expire_on_commit=False): return session -def create_coverage(project_name, coverage_rate=0.0, report_time=None, - test_type='py27', session=None): +def create_coverage(project_name, coverage_rate=0.0, rates=[], + report_time=None, test_type='py27', session=None): """Create a new coverage record in the database. This method is used to add a new coverage in the database. @@ -98,12 +98,41 @@ def create_coverage(project_name, coverage_rate=0.0, report_time=None, report_time_microsecond = None coverage.report_time = report_time coverage.report_time_microsecond = report_time_microsecond + session = session or get_session() with session.begin(): session.add(coverage) + return coverage +def add_file_rates(coverage_id, rates=[], session=None): + """Add rates a specific coverage. + + This method is used to add rate records in the database. + It tracks the coverage history by individual files. + + :param int coverage_id: coverage_id + :param list rates: rates dict list which has + e.g. [{'filename': 'foo', 'line-rate': '0.1'}, ...] + :param session: optional session object if one isn't provided a new session + will be acquired for the duration of this operation + :return: The list of created files objects + :rtype: coverage2sql.models.File + """ + session = session or get_session() + files = [] + with session.begin(): + for r in rates: + f = models.File() + f.coverage_id = coverage_id + f.filename = r['filename'] + f.line_rate = r['line-rate'] + session.add(f) + files.append(f) + return files + + def get_coverage(project_name=None, test_type=None, session=None): """Get new coverage records in the database. diff --git a/coverage2sql/db/models.py b/coverage2sql/db/models.py index 5533085..c33a598 100644 --- a/coverage2sql/db/models.py +++ b/coverage2sql/db/models.py @@ -54,3 +54,17 @@ class Coverage(BASE, CoverageBase): test_type = sa.Column(sa.String(256), nullable=False, default='py27') report_time = sa.Column(sa.DateTime(), default=datetime.datetime.utcnow()) report_time_microsecond = sa.Column(sa.Integer(), default=0) + + +class File(BASE, CoverageBase): + __tablename__ = 'files' + __table_args__ = (sa.Index('ix_file_coverage_id', 'coverage_id'), + sa.Index('ix_filename', 'filename')) + id = sa.Column(sa.BigInteger, primary_key=True) + coverage_id = sa.Column(sa.BigInteger, nullable=False) + filename = sa.Column(sa.String(256), nullable=False) + line_rate = sa.Column(sa.Float()) + coverage = sa.orm.relationship(Coverage, + backref=sa.orm.backref('file_coverage'), + foreign_keys=coverage_id, + primaryjoin=coverage_id == Coverage.id) diff --git a/coverage2sql/migrations/versions/52dfb338f74e_add_coverages_table.py b/coverage2sql/migrations/versions/52dfb338f74e_add_coverages_table.py index dd7349b..c4ebc94 100644 --- a/coverage2sql/migrations/versions/52dfb338f74e_add_coverages_table.py +++ b/coverage2sql/migrations/versions/52dfb338f74e_add_coverages_table.py @@ -52,4 +52,4 @@ def upgrade(): def downgrade(): - raise NotImplementedError() + op.drop_table('classes') diff --git a/coverage2sql/migrations/versions/79dead6f7c26_add_files_table.py b/coverage2sql/migrations/versions/79dead6f7c26_add_files_table.py new file mode 100644 index 0000000..0bf39fd --- /dev/null +++ b/coverage2sql/migrations/versions/79dead6f7c26_add_files_table.py @@ -0,0 +1,54 @@ +# Copyright (c) 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add files table + +Revision ID: 79dead6f7c26 +Revises: cb0e61ce633e +Create Date: 2017-02-07 17:57:28.777311 + +""" + +# revision identifiers, used by Alembic. +revision = '79dead6f7c26' +down_revision = 'cb0e61ce633e' +branch_labels = None +depends_on = None + +from alembic import context +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + migration_context = context.get_context() + if migration_context.dialect.name == 'sqlite': + id_type = sa.Integer + else: + id_type = sa.BigInteger + + op.create_table('files', + sa.Column('id', id_type, autoincrement=True, + primary_key=True), + sa.Column('coverage_id', id_type, nullable=False), + sa.Column('filename', sa.String(256), nullable=False), + sa.Column('line_rate', sa.Float()), + mysql_engine='InnoDB') + op.create_index('ix_class_coverage_id', 'files', ['coverage_id']) + op.create_index('ix_filename', 'files', ['filename']) + + +def downgrade(): + op.drop_table('files') diff --git a/coverage2sql/read_coverage.py b/coverage2sql/read_coverage.py index 7aa1979..8a9aa53 100644 --- a/coverage2sql/read_coverage.py +++ b/coverage2sql/read_coverage.py @@ -12,7 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. +import tempfile + import coverage +import xmltodict class DevNull(object): @@ -26,10 +29,30 @@ class ReadCoverage(object): def __init__(self, coverage_file=None): self.cov = coverage.Coverage(data_file=coverage_file) self.cov.load() - self.cov_pct = self.cov.report(file=DevNull()) + xmlfile = tempfile.NamedTemporaryFile(suffix='.xml') + self.cov_pct = self.cov.xml_report(outfile=xmlfile.name) + self.covdoc = xmltodict.parse(xmlfile.read()) + xmlfile.close() def get_data(self): return self.cov.get_data() def get_coverage_rate(self): return self.cov_pct / 100 + + def get_rates_by_files(self): + rates = [] + for p in self.covdoc['coverage']['packages']['package']: + try: + # FIXME(masayukig): Try to access the first element. This is + # ugly.. + p['classes']['class'][0] + for c in p['classes']['class']: + rates.append({'filename': c['@filename'], + 'line-rate': c['@line-rate']}) + except KeyError: + # NOTE(masayukig): This has only one class + c = p['classes']['class'] + rates.append({'filename': c['@filename'], + 'line-rate': c['@line-rate']}) + return rates diff --git a/coverage2sql/shell.py b/coverage2sql/shell.py index 0bb6925..15d12c4 100644 --- a/coverage2sql/shell.py +++ b/coverage2sql/shell.py @@ -62,9 +62,11 @@ def parse_args(argv, default_config_files=None): default_config_files=default_config_files) -def process_results(project_name=".", coverage_rate=0.0): +def process_results(project_name=".", coverage_rate=0.0, rates=[]): session = api.get_session() - api.create_coverage(project_name, coverage_rate, test_type=CONF.test_type) + cov = api.create_coverage(project_name, coverage_rate, rates, + test_type=CONF.test_type, session=session) + api.add_file_rates(cov.id, rates, session) session.close() @@ -76,9 +78,10 @@ def main(): if CONF.coverage_file: cov = coverage.ReadCoverage(CONF.coverage_file) coverage_rate = cov.get_coverage_rate() + rates = cov.get_rates_by_files() else: raise NotImplementedError() - process_results(project_name, coverage_rate) + process_results(project_name, coverage_rate, rates) if __name__ == "__main__": diff --git a/coverage2sql/tests/db/test_api.py b/coverage2sql/tests/db/test_api.py index 90e0349..c8692da 100644 --- a/coverage2sql/tests/db/test_api.py +++ b/coverage2sql/tests/db/test_api.py @@ -65,3 +65,14 @@ class TestDatabaseAPI(base.TestCase): self.assertTrue(covs is not None) self.assertEqual(len(covs), 1) self.assertEqual(covs[0].project_name, 'foo1_project') + + def test_add_file_rates(self): + rates = [] + rates.append({'filename': 'foo/bar0', 'line-rate': '0'}) + rates.append({'filename': 'foo/bar1', 'line-rate': '1'}) + rates.append({'filename': 'foo/bar2', 'line-rate': '0.92'}) + files = api.add_file_rates(1, rates) + self.assertEqual(3, len(files)) + for r, f in zip(rates, files): + self.assertEqual(r['filename'], f.filename) + self.assertEqual(r['line-rate'], f.line_rate) diff --git a/coverage2sql/tests/test_shell.py b/coverage2sql/tests/test_shell.py index 1c3cc02..5e54676 100644 --- a/coverage2sql/tests/test_shell.py +++ b/coverage2sql/tests/test_shell.py @@ -44,6 +44,10 @@ class TestMain(base.TestCase): 'get_coverage_rate') fake_read_coverage.get_coverage_rate.return_value = ( fake_get_coverage_rate) + fake_read_coverage.get_rates_by_files = mock.MagicMock( + 'get_rates_by_files') + fake_read_coverage.get_rates_by_files.return_value = ( + {'filename': 'foo/bar.py', 'line-rate': '0.99'}) read_coverage_mock.return_value = fake_read_coverage shell.main() read_coverage_mock.assert_called_with(mock.ANY) diff --git a/releasenotes/notes/add-files-table-e60ac2e98b5f543b.yaml b/releasenotes/notes/add-files-table-e60ac2e98b5f543b.yaml new file mode 100644 index 0000000..ff8b7d5 --- /dev/null +++ b/releasenotes/notes/add-files-table-e60ac2e98b5f543b.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Add 'Files' table to store individual coverage data by file. diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 78e8ca8..ecc034f 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -1,14 +1,13 @@ Welcome to Coverage2sql Release Notes documentation! -=================================================== +==================================================== Contents ======== -.. toctree:: - :maxdepth: 2 - - unreleased + .. toctree:: + :maxdepth: 1 + unreleased Indices and tables ================== diff --git a/requirements.txt b/requirements.txt index f0a3376..6ea5d57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,5 @@ pbr>=1.6 SQLAlchemy>=0.8.2 alembic>=0.4.1 oslo.config>=1.4.0.0a3 +coverage>=4.3 +xmltodict>=0.10.1 # MIT diff --git a/test-requirements.txt b/test-requirements.txt index 3e23420..113c6eb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,7 +4,6 @@ hacking<0.11,>=0.10.0 -coverage>=3.6 docutils>=0.11 # OSI-Approved Open Source, Public Domain fixtures>=0.3.14 python-subunit>=0.0.18