ara/ara/models.py

#  Copyright (c) 2017 Red Hat, Inc.
#
#  This file is part of ARA: Ansible Run Analysis.
#
#  ARA is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  ARA is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with ARA.  If not, see <http://www.gnu.org/licenses/>.

import functools
import hashlib
import uuid
import zlib

from datetime import datetime
from datetime import timedelta
from oslo_utils import encodeutils
from oslo_serialization import jsonutils

# This makes all the exceptions available as "models.<exception_name>".
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy.orm.exc import *  # NOQA
from sqlalchemy.orm import backref
import sqlalchemy.types as types

db = SQLAlchemy()


def mkuuid():
    """
    This is used to generate primary keys in the database tables.
    We were simply passing `default=uuid.uuid4` to `db.Column`, but it turns
    out that while some database drivers seem to implicitly call `str()`,
    others may be calling `repr()` which resulted in SQLIte trying to
    use keys like `UUID('a496d538-c819-4f7c-8926-e3abe317239d')`.
    """
    return str(uuid.uuid4())


def content_sha1(context):
    """
    Used by the FileContent model to automatically compute the sha1
    hash of content before storing it to the database.
    """
    try:
        content = context.current_parameters['content']
    except AttributeError:
        content = context
    return hashlib.sha1(encodeutils.to_utf8(content)).hexdigest()


# Primary key columns are of these type.
pkey_type = db.String(36)

# This defines the standard primary key column used in our tables.
std_pkey = functools.partial(
    db.Column, pkey_type, primary_key=True,
    nullable=False, default=mkuuid)

# Common options for one-to-one relationships in our database.
one_to_one = functools.partial(
    db.relationship, passive_deletes=False,
    cascade='all, delete-orphan', uselist=False)

# common options for one-to-many relationships in our database.
one_to_many = functools.partial(
    db.relationship, passive_deletes=False,
    cascade='all, delete-orphan', lazy='dynamic')

# common options for many-to-one relationships in our database.
many_to_one = functools.partial(
    db.relationship, passive_deletes=False,
    cascade='all, delete-orphan',
    single_parent=True)

# Common options for many-to-many relationships in our database.
many_to_many = functools.partial(
    db.relationship, passive_deletes=False,
    cascade='all, delete',
    lazy='dynamic')


# Common options for foreign key relationships.
def std_fkey(col):
    return db.Column(pkey_type, db.ForeignKey(col, ondelete='RESTRICT'))


class TimedEntity(object):
    @property
    def duration(self):
        """
        Calculates '(time_end-time_start)' and return the resulting
        'datetime.timedelta' object.
        """
        if self.time_end is None or self.time_start is None:
            return timedelta(seconds=0)
        else:
            return self.time_end - self.time_start

    def start(self):
        """
        Explicitly sets 'self.time_start'
        """
        self.time_start = datetime.now()

    def stop(self):
        """
        Explicitly sets 'self.time_end'
        """
        self.time_end = datetime.now()


class CompressedData(types.TypeDecorator):
    """
    Implements a new sqlalchemy column type that automatically serializes
    and compresses data when writing it to the database and decompresses
    the data when reading it.

    http://docs.sqlalchemy.org/en/latest/core/custom_types.html
    """
    impl = types.Binary

    def process_bind_param(self, value, dialect):
        return zlib.compress(encodeutils.to_utf8(jsonutils.dumps(value)))

    def process_result_value(self, value, dialect):
        if value is not None:
            return jsonutils.loads(zlib.decompress(value))
        else:
            return value

    def copy(self, **kwargs):
        return CompressedData(self.impl.length)


class CompressedText(types.TypeDecorator):
    """
    Implements a new sqlalchemy column type that automatically compresses
    data when writing it to the database and decompresses the data when
    reading it.

    http://docs.sqlalchemy.org/en/latest/core/custom_types.html
    """
    impl = types.Binary

    def process_bind_param(self, value, dialect):
        return zlib.compress(encodeutils.to_utf8(value))

    def process_result_value(self, value, dialect):
        return encodeutils.safe_decode(zlib.decompress(value))

    def copy(self, **kwargs):
        return CompressedText(self.impl.length)


class Playbook(db.Model, TimedEntity):
    """
    The 'Playbook' class represents a single run of 'ansible-playbook'.

    'Playbook' entities have the following relationships:
    - 'data' -- a list of k/v pairs recorded in this playbook run
    - 'plays' -- a list of plays encountered in this playbook run
    - 'tasks' -- a list of tasks encountered in this playbook run
    - 'stats' -- a list of  statistic records, one for each host
      involved in this playbook
    - 'hosts' -- a list of hosts involved in this playbook
    - 'files' -- a list of files encountered by this playbook
      (via include or role directives).
    """
    __tablename__ = 'playbooks'

    id = std_pkey()
    path = db.Column(db.String(255))
    ansible_version = db.Column(db.String(255))
    options = db.Column(CompressedData((2 ** 32) - 1))

    data = one_to_many('Data', backref='playbook')
    files = one_to_many('File', backref='playbook')
    plays = one_to_many('Play', backref='playbook')
    tasks = one_to_many('Task', backref='playbook')
    stats = one_to_many('Stats', backref='playbook')
    hosts = one_to_many('Host', backref='playbook')

    time_start = db.Column(db.DateTime, default=datetime.now)
    time_end = db.Column(db.DateTime)

    complete = db.Column(db.Boolean, default=False)

    @property
    def file(self):
        return (self.files
                .filter(File.playbook_id == self.id)
                .filter(File.is_playbook)).one()

    def __repr__(self):
        return '<Playbook %s>' % self.path


class File(db.Model):
    """
    Represents a task list (role or playbook or included file) referenced by
    an Ansible run.
    """
    __tablename__ = 'files'
    __table_args__ = (
        db.UniqueConstraint('playbook_id', 'path'),
    )

    id = std_pkey()
    playbook_id = std_fkey('playbooks.id')

    # This has to be a String instead of Text because of
    # http://stackoverflow.com/questions/1827063/
    # and it must have a max length smaller than PATH_MAX because MySQL is
    # limited to a maximum key length of 3072 bytes. These restrictions stem
    # from the fact that we are using this column in a UNIQUE constraint.
    path = db.Column(db.String(255))
    content = many_to_one('FileContent', backref='files')
    content_id = db.Column(db.String(40),
                           db.ForeignKey('file_contents.id'))

    tasks = many_to_one('Task', backref=backref('file', uselist=False))

    # is_playbook is true for playbooks referenced directly on the
    # ansible-playbook command line.
    is_playbook = db.Column(db.Boolean, default=False)


class FileContent(db.Model):
    """
    Stores content of Ansible task lists encountered during an Ansible run.
    We store content keyed by the its sha1 hash, so if a file doesn't change
    the content will only be stored once in the database. The hash is
    calculated automatically when the object is written to the database.
    """
    __tablename__ = 'file_contents'

    id = db.Column(db.String(40), primary_key=True, default=content_sha1)
    content = db.Column(CompressedText((2**32) - 1))


class Play(db.Model, TimedEntity):
    """
    The 'Play' class represents a play in an ansible playbook.

    'Play' entities have the following relationships:
    - 'tasks' -- a list of tasks in this play
    - 'task_results' -- a list of task results in this play (via the
      'tasks' relationship defined by 'TaskResult').
    """
    __tablename__ = 'plays'

    id = std_pkey()
    playbook_id = std_fkey('playbooks.id')
    name = db.Column(db.Text)
    sortkey = db.Column(db.Integer)
    tasks = one_to_many('Task', backref='play')

    time_start = db.Column(db.DateTime, default=datetime.now)
    time_end = db.Column(db.DateTime)

    def __repr__(self):
        return '<Play %s>' % (self.name or self.id)

    @property
    def offset_from_playbook(self):
        return self.time_start - self.playbook.time_start


class Task(db.Model, TimedEntity):
    """
    The 'Task' class represents a single task defined in an Ansible playbook.

    'Task' entities have the following relationships:
    - 'playbook' -- the playbook containing this task (via the 'tasks'
      relationship defined by 'Playbook')
    - 'play' -- the play containing this task (via the 'tasks' relationship
      defined by 'Play')
    - 'task_results' -- a list of results for each host targeted by this task.
    """
    __tablename__ = 'tasks'

    id = std_pkey()
    playbook_id = std_fkey('playbooks.id')
    play_id = std_fkey('plays.id')

    name = db.Column(db.Text)
    sortkey = db.Column(db.Integer)
    action = db.Column(db.Text)
    tags = db.Column(db.Text)
    is_handler = db.Column(db.Boolean)

    file_id = std_fkey('files.id')
    lineno = db.Column(db.Integer)

    time_start = db.Column(db.DateTime, default=datetime.now)
    time_end = db.Column(db.DateTime)

    task_results = one_to_many('TaskResult', backref='task')

    def __repr__(self):
        return '<Task %s>' % (self.name or self.id)

    @property
    def offset_from_playbook(self):
        return self.time_start - self.playbook.time_start

    @property
    def offset_from_play(self):
        return self.time_start - self.play.time_start


class TaskResult(db.Model, TimedEntity):
    """
    The 'TaskResult' class represents the result of running a single task on
    a single host.

    A 'TaskResult' entity has the following relationships:
    - 'task' -- the task for which this is a result (via the 'task_results'
      relationship defined by 'Task').
    - 'host' -- the host associated with this result (via the 'task_results'
      relationship defined by 'Host')
    """
    __tablename__ = 'task_results'

    id = std_pkey()
    task_id = std_fkey('tasks.id')
    host_id = std_fkey('hosts.id')

    status = db.Column(db.Enum('ok', 'failed', 'skipped', 'unreachable'))
    changed = db.Column(db.Boolean, default=False)
    failed = db.Column(db.Boolean, default=False)
    skipped = db.Column(db.Boolean, default=False)
    unreachable = db.Column(db.Boolean, default=False)
    ignore_errors = db.Column(db.Boolean, default=False)
    result = db.Column(CompressedText((2**32) - 1))

    time_start = db.Column(db.DateTime, default=datetime.now)
    time_end = db.Column(db.DateTime)

    @property
    def derived_status(self):
        if self.status == 'ok' and self.changed:
            return 'changed'
        elif self.status == 'failed' and self.ignore_errors:
            return 'ignored'
        else:
            return self.status

    def __repr__(self):
        return '<TaskResult %s>' % self.host.name


class Host(db.Model):
    """
    The 'Host' object represents a host reference by an Ansible inventory.

    A 'Host' entity has the following relationships:
    - 'task_results' -- a list of 'TaskResult' objects associated with this
      host.
    - 'stats' -- a list of 'Stats' objects resulting from playbook runs
      against this host.
    - 'playbooks' -- a list of 'Playbook' runs that have included this host.
    """
    __tablename__ = 'hosts'
    __table_args__ = (
        db.UniqueConstraint('playbook_id', 'name'),
    )

    id = std_pkey()
    playbook_id = std_fkey('playbooks.id')
    name = db.Column(db.String(255), index=True)

    facts = one_to_one('HostFacts', backref='host')
    task_results = one_to_many('TaskResult', backref='host')
    stats = one_to_one('Stats', backref='host')

    def __repr__(self):
        return '<Host %s>' % self.name


class HostFacts(db.Model):
    """
    The 'HostFacts' object represents a host reference by an Ansible
    inventory. It is meant to record facts when a setup task is run for a host.

    A 'HostFacts' entity has the following relationship:
    - 'hosts' -- the host owner of the facts
    """
    __tablename__ = 'host_facts'

    id = std_pkey()
    host_id = std_fkey('hosts.id')
    timestamp = db.Column(db.DateTime, default=datetime.now)
    values = db.Column(db.Text(16777215))

    def __repr__(self):
        return '<HostFacts %s>' % self.host.name


class Stats(db.Model):
    """
    A 'Stats' object contains statistics for a single host from a single
    Ansible playbook run.

    A 'Stats' entity has the following relationships:
    - 'playbook' -- the playbook associated with these statistics (via the
      'stats' relationship defined in `Playbook`)
    - 'host' -- The host associated with these statistics (via the
      'stats' relationship defined in 'Host')
    """
    __tablename__ = 'stats'

    id = std_pkey()
    playbook_id = std_fkey('playbooks.id')
    host_id = std_fkey('hosts.id')

    changed = db.Column(db.Integer, default=0)
    failed = db.Column(db.Integer, default=0)
    ok = db.Column(db.Integer, default=0)
    skipped = db.Column(db.Integer, default=0)
    unreachable = db.Column(db.Integer, default=0)

    def __repr__(self):
        return '<Stats for %s>' % self.host.name


class Data(db.Model):
    """
    The 'Data' object represents a recorded key/value pair provided by the
    ara_record module.

    A 'Data' entity has the following relationships:
    - 'playbook' -- the playbook this key/value pair was recorded in
    """
    __tablename__ = 'data'
    __table_args__ = (
        db.UniqueConstraint('playbook_id', 'key'),
    )

    id = std_pkey()
    playbook_id = std_fkey('playbooks.id')
    key = db.Column(db.String(255))
    value = db.Column(CompressedData((2 ** 32) - 1))
    type = db.Column(db.String(255))

    def __repr__(self):
        return '<Data %s:%s>' % (self.data.playbook_id, self.data.key)