diff --git a/trove/common/cfg.py b/trove/common/cfg.py index 536eaad952..38a404b86a 100644 --- a/trove/common/cfg.py +++ b/trove/common/cfg.py @@ -524,6 +524,41 @@ mongodb_opts = [ 'logic.'), ] +# PostgreSQL +postgresql_group = cfg.OptGroup( + 'postgresql', title='PostgreSQL options', + help="Oslo option group for the PostgreSQL datastore.") +postgresql_opts = [ + cfg.ListOpt('tcp_ports', default=["5432"], + help='List of TCP ports and/or port ranges to open' + ' in the security group (only applicable ' + 'if trove_security_groups_support is True).'), + cfg.ListOpt('udp_ports', default=[], + help='List of UPD ports and/or port ranges to open' + ' in the security group (only applicable ' + 'if trove_security_groups_support is True).'), + cfg.StrOpt('backup_strategy', default='PgDump', + help='Default strategy to perform backups.'), + cfg.StrOpt('mount_point', default='/var/lib/postgresql', + help="Filesystem path for mounting " + "volumes if volume support is enabled."), + cfg.BoolOpt('root_on_create', default=False, + help='Enable the automatic creation of the root user for the ' + 'service during instance-create. The generated password for ' + 'the root user is immediately returned in the response of ' + "instance-create as the 'password' field."), + cfg.StrOpt('backup_namespace', + default='trove.guestagent.strategies.backup.postgresql_impl'), + cfg.StrOpt('restore_namespace', + default='trove.guestagent.strategies.restore.postgresql_impl'), + cfg.BoolOpt('volume_support', + default=True, + help='Whether to provision a cinder volume for datadir.'), + cfg.StrOpt('device_path', default='/dev/vdb'), + cfg.ListOpt('ignore_users', default=['os_admin', 'postgres', 'root']), + cfg.ListOpt('ignore_dbs', default=['postgres']), +] + CONF = cfg.CONF CONF.register_opts(path_opts) @@ -542,6 +577,7 @@ CONF.register_opts(redis_opts, redis_group) CONF.register_opts(cassandra_opts, cassandra_group) CONF.register_opts(couchbase_opts, couchbase_group) CONF.register_opts(mongodb_opts, mongodb_group) +CONF.register_opts(postgresql_opts, postgresql_group) def custom_parser(parsername, parser): diff --git a/trove/guestagent/datastore/postgresql/__init__.py b/trove/guestagent/datastore/postgresql/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/trove/guestagent/datastore/postgresql/manager.py b/trove/guestagent/datastore/postgresql/manager.py new file mode 100644 index 0000000000..8a4b2b08ce --- /dev/null +++ b/trove/guestagent/datastore/postgresql/manager.py @@ -0,0 +1,98 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import os +from trove.common import cfg +from trove.guestagent import dbaas +from trove.guestagent import backup +from trove.guestagent import volume +from .service.config import PgSqlConfig +from .service.database import PgSqlDatabase +from .service.install import PgSqlInstall +from .service.root import PgSqlRoot +from .service.users import PgSqlUsers +from .service.status import PgSqlAppStatus +from trove.openstack.common import log as logging +from trove.openstack.common import periodic_task + + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class Manager( + periodic_task.PeriodicTasks, + PgSqlUsers, + PgSqlDatabase, + PgSqlRoot, + PgSqlConfig, + PgSqlInstall, +): + + def __init__(self, *args, **kwargs): + super(Manager, self).__init__(*args, **kwargs) + + @periodic_task.periodic_task(ticks_between_runs=3) + def update_status(self, context): + PgSqlAppStatus.get().update() + + def prepare( + self, + context, + packages, + databases, + memory_mb, + users, + device_path=None, + mount_point=None, + backup_info=None, + config_contents=None, + root_password=None, + overrides=None, + ): + self.install(context, packages) + PgSqlAppStatus.get().begin_restart() + self.stop_db(context) + if device_path: + device = volume.VolumeDevice(device_path) + device.format() + if os.path.exists(mount_point): + if not backup_info: + device.migrate_data(mount_point) + device.mount(mount_point) + self.reset_configuration(context, config_contents) + self.set_db_to_listen(context) + self.start_db(context) + + if backup_info: + backup.restore(context, backup_info, '/tmp') + + if root_password and not backup_info: + self.enable_root(context, root_password) + + PgSqlAppStatus.get().end_install_or_restart() + + if databases: + self.create_database(context, databases) + + if users: + self.create_user(context, users) + + def get_filesystem_stats(self, context, fs_path): + return dbaas.get_filesystem_volume_stats(fs_path) + + def create_backup(self, context, backup_info): + backup.backup(context, backup_info) diff --git a/trove/guestagent/datastore/postgresql/pgutil.py b/trove/guestagent/datastore/postgresql/pgutil.py new file mode 100644 index 0000000000..96deb840d5 --- /dev/null +++ b/trove/guestagent/datastore/postgresql/pgutil.py @@ -0,0 +1,229 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import tempfile +import uuid + +from trove.common import utils +from trove.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + + +def execute(*command, **kwargs): + """Execute a command as the 'postgres' user.""" + + LOG.debug('Running as postgres: {0}'.format(command)) + return utils.execute_with_timeout( + "sudo", "-u", "postgres", *command, **kwargs + ) + + +def result(filename): + """A generator representing the results of a query. + + This generator produces result records of a query by iterating over a + CSV file created by the query. When the file is out of records it is + removed. + + The purpose behind this abstraction is to provide a record set interface + with minimal memory consumption without requiring an active DB connection. + This makes it possible to iterate over any sized record set without + allocating memory for the entire record set and without using a DB cursor. + + Each row is returned as an iterable of column values. The order of these + values is determined by the query. + """ + + utils.execute_with_timeout( + 'sudo', 'chmod', '777', filename, + ) + with open(filename, 'r+') as file_handle: + for line in file_handle: + if line != "": + yield line.split(',') + execute( + "rm", "{filename}".format(filename=filename), + ) + raise StopIteration() + + +def psql(statement, timeout=30): + """Execute a statement using the psql client.""" + + LOG.debug('Sending to local db: {0}'.format(statement)) + return execute('psql', '-c', statement, timeout=timeout) + + +def query(statement, timeout=30): + """Execute a pgsql query and get a generator of results. + + This method will pipe a CSV format of the query results into a temporary + file. The return value is a generator object that feeds from this file. + """ + + filename = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) + LOG.debug('Querying: {0}'.format(statement)) + psql( + "Copy ({statement}) To '{filename}' With CSV".format( + statement=statement, + filename=filename, + ), + timeout=timeout, + ) + + return result(filename) + + +class DatabaseQuery(object): + + @classmethod + def list(cls, ignore=()): + """Query to list all databases.""" + + statement = ( + "SELECT datname, pg_encoding_to_char(encoding), " + "datcollate FROM pg_database " + "WHERE datistemplate = false" + ) + + for name in ignore: + statement += " AND datname != '{name}'".format(name=name) + + return statement + + @classmethod + def create(cls, name, encoding=None, collation=None): + """Query to create a database.""" + + statement = "CREATE DATABASE {name}".format(name=name) + if encoding is not None: + statement += " ENCODING = {encoding}".format( + encoding=encoding, + ) + if collation is not None: + statement += " LC_COLLATE = {collation}".format( + collation=collation, + ) + + return statement + + @classmethod + def drop(cls, name): + """Query to drop a database.""" + + return "DROP DATABASE IF EXISTS {name}".format(name=name) + + +class UserQuery(object): + + @classmethod + def list(cls, ignore=()): + """Query to list all users.""" + + statement = "SELECT usename FROM pg_catalog.pg_user" + if ignore: + # User a simple tautology so all clauses can be AND'ed without + # crazy special logic. + statement += " WHERE 1=1" + for name in ignore: + statement += " AND usename != '{name}'".format(name=name) + + return statement + + @classmethod + def list_root(cls, ignore=()): + """Query to list all superuser accounts.""" + + statement = ( + "SELECT usename FROM pg_catalog.pg_user WHERE usesuper = true" + ) + + for name in ignore: + statement += " AND usename != '{name}'".format(name=name) + + return statement + + @classmethod + def get(cls, name): + """Query to get a single user.""" + + return ( + "SELECT usename FROM pg_catalog.pg_user " + "WHERE usename = '{name}'".format(name=name) + ) + + @classmethod + def create(cls, name, password): + """Query to create a user with a password.""" + + return "CREATE USER {name} WITH PASSWORD '{password}'".format( + name=name, + password=password, + ) + + @classmethod + def update_password(cls, name, password): + """Query to update the password for a user.""" + + return "ALTER USER {name} WITH PASSWORD '{password}'".format( + name=name, + password=password, + ) + + @classmethod + def update_name(cls, old, new): + """Query to update the name of a user.""" + + return "ALTER USER {old} RENAME TO {new}".format(old=old, new=new) + + @classmethod + def drop(cls, name): + """Query to drop a user.""" + + return "DROP USER {name}".format(name=name) + + +class AccessQuery(object): + + @classmethod + def list(cls, user): + """Query to list grants for a user.""" + + return ( + "SELECT datname " + "FROM pg_database " + "WHERE datistemplate = false " + "AND 'user {user}=CTc' = ANY (datacl)".format(user=user) + ) + + @classmethod + def grant(cls, user, database): + """Query to grant user access to a database.""" + + return "GRANT ALL ON DATABASE {database} TO {user}".format( + database=database, + user=user, + ) + + @classmethod + def revoke(cls, user, database): + """Query to revoke user access to a database.""" + + return "REVOKE ALL ON DATABASE {database} FROM {user}".format( + database=database, + user=user, + ) diff --git a/trove/guestagent/datastore/postgresql/service/__init__.py b/trove/guestagent/datastore/postgresql/service/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/trove/guestagent/datastore/postgresql/service/access.py b/trove/guestagent/datastore/postgresql/service/access.py new file mode 100644 index 0000000000..77d5ca7136 --- /dev/null +++ b/trove/guestagent/datastore/postgresql/service/access.py @@ -0,0 +1,94 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from trove.common import cfg +from trove.guestagent.datastore.postgresql import pgutil +from trove.openstack.common import log as logging +from trove.openstack.common.gettextutils import _ + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class PgSqlAccess(object): + """Mixin implementing the user-access API calls.""" + + def grant_access(self, context, username, hostname, databases): + """Give a user permission to use a given database. + + The username and hostname parameters are strings. + The databases parameter is a list of strings representing the names of + the databases to grant permission on. + """ + for database in databases: + LOG.info( + _("{guest_id}: Granting user ({user}) access to database " + "({database}).").format( + guest_id=CONF.guest_id, + user=username, + database=database, + ) + ) + pgutil.psql( + pgutil.AccessQuery.grant( + user=username, + database=database, + ), + timeout=30, + ) + + def revoke_access(self, context, username, hostname, database): + """Revoke a user's permission to use a given database. + + The username and hostname parameters are strings. + The database parameter is a string representing the name of the + database. + """ + LOG.info( + _("{guest_id}: Revoking user ({user}) access to database" + "({database}).").format( + guest_id=CONF.guest_id, + user=username, + database=database, + ) + ) + pgutil.psql( + pgutil.AccessQuery.revoke( + user=username, + database=database, + ), + timeout=30, + ) + + def list_access(self, context, username, hostname): + """List database for which the given user as access. + + The username and hostname parameters are strings. + + Return value is a list of dictionaries in the following form: + + [{"_name": "", "_collate": None, "_character_set": None}, ...] + """ + results = pgutil.query( + pgutil.AccessQuery.list(user=username), + timeout=30, + ) + + # Convert to dictionaries. + results = ( + {'_name': r[0].strip(), '_collate': None, '_character_set': None} + for r in results + ) + return tuple(results) diff --git a/trove/guestagent/datastore/postgresql/service/config.py b/trove/guestagent/datastore/postgresql/service/config.py new file mode 100644 index 0000000000..76439078a5 --- /dev/null +++ b/trove/guestagent/datastore/postgresql/service/config.py @@ -0,0 +1,127 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re +from trove.common import cfg +from trove.common import utils +from trove.guestagent.datastore.postgresql import pgutil +from trove.guestagent.datastore.postgresql.service.process import PgSqlProcess +from trove.guestagent.datastore.postgresql.service.status import PgSqlAppStatus +from trove.openstack.common import log as logging +from trove.openstack.common.gettextutils import _ + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + +PGSQL_CONFIG = "/etc/postgresql/{version}/main/postgresql.conf" +PGSQL_HBA_CONFIG = "/etc/postgresql/{version}/main/pg_hba.conf" + + +class PgSqlConfig(PgSqlProcess): + """Mixin that implements the config API. + + This mixin has a dependency on the PgSqlProcess mixin. + """ + + def _get_psql_version(self): + """Poll PgSql for the version number. + + Return value is a string representing the version number. + """ + LOG.debug( + "{guest_id}: Polling for postgresql version.".format( + guest_id=CONF.guest_id, + ) + ) + out, err = pgutil.execute('psql', '--version', timeout=30) + pattern = re.compile('\d\.\d') + return pattern.search(out).group(0) + + def reset_configuration(self, context, configuration): + """Reset the PgSql configuration file to the one given. + + The configuration parameter is a string containing the full + configuration file that should be used. + """ + config_location = PGSQL_CONFIG.format( + version=self._get_psql_version(), + ) + LOG.debug( + "{guest_id}: Writing configuration file to /tmp/pgsql_config." + .format( + guest_id=CONF.guest_id, + ) + ) + with open('/tmp/pgsql_config', 'w+') as config_file: + config_file.write(configuration) + utils.execute_with_timeout( + 'sudo', 'chown', 'postgres', '/tmp/pgsql_config', + timeout=30, + ) + utils.execute_with_timeout( + 'sudo', 'mv', '/tmp/pgsql_config', config_location, + timeout=30, + ) + + def set_db_to_listen(self, context): + """Allow remote connections with encrypted passwords.""" + # Using cat to read file due to read permissions issues. + out, err = utils.execute_with_timeout( + 'sudo', 'cat', + PGSQL_HBA_CONFIG.format( + version=self._get_psql_version(), + ), + timeout=30, + ) + LOG.debug( + "{guest_id}: Writing hba file to /tmp/pgsql_hba_config.".format( + guest_id=CONF.guest_id, + ) + ) + with open('/tmp/pgsql_hba_config', 'w+') as config_file: + config_file.write(out) + config_file.write("host all all 0.0.0.0/0 md5\n") + + utils.execute_with_timeout( + 'sudo', 'chown', 'postgres', '/tmp/pgsql_hba_config', + timeout=30, + ) + utils.execute_with_timeout( + 'sudo', 'mv', '/tmp/pgsql_hba_config', + PGSQL_HBA_CONFIG.format( + version=self._get_psql_version(), + ), + timeout=30, + ) + + def start_db_with_conf_changes(self, context, config_contents): + """Restarts the PgSql instance with a new configuration.""" + LOG.info( + _("{guest_id}: Going into restart mode for config file changes.") + .format( + guest_id=CONF.guest_id, + ) + ) + PgSqlAppStatus.get().begin_restart() + self.stop_db(context) + self.reset_configuration(context, config_contents) + self.start_db(context) + LOG.info( + _("{guest_id}: Ending restart mode for config file changes.") + .format( + guest_id=CONF.guest_id, + ) + ) + PgSqlAppStatus.get().end_install_or_restart() diff --git a/trove/guestagent/datastore/postgresql/service/database.py b/trove/guestagent/datastore/postgresql/service/database.py new file mode 100644 index 0000000000..2ccc8c7064 --- /dev/null +++ b/trove/guestagent/datastore/postgresql/service/database.py @@ -0,0 +1,121 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools +from trove.common import cfg +from trove.guestagent.datastore.postgresql import pgutil +from trove.openstack.common import log as logging +from trove.openstack.common.gettextutils import _ + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF +IGNORE_DBS_LIST = CONF.get(CONF.datastore_manager).ignore_dbs + + +class PgSqlDatabase(object): + + def create_database(self, context, databases): + """Create the list of specified databases. + + The databases parameter is a list of dictionaries in the following + form: + + {"_name": "", "_character_set": "", "_collate": ""} + + Encoding and collation values are validated in + trove.guestagent.db.models. + """ + for database in databases: + encoding = database.get('_character_set') + collate = database.get('_collate') + LOG.info( + _("{guest_id}: Creating database {name}.").format( + guest_id=CONF.guest_id, + name=database['_name'], + ) + ) + pgutil.psql( + pgutil.DatabaseQuery.create( + name=database['_name'], + encoding=encoding, + collation=collate, + ), + timeout=30, + ) + + def delete_database(self, context, database): + """Delete the specified database. + + The database parameter is a dictionary in the following form: + + {"_name": ""} + """ + LOG.info( + _("{guest_id}: Dropping database {name}.").format( + guest_id=CONF.guest_id, + name=database['_name'], + ) + ) + pgutil.psql( + pgutil.DatabaseQuery.drop(name=database['_name']), + timeout=30, + ) + + def list_databases( + self, + context, + limit=None, + marker=None, + include_marker=False, + ): + """List databases created on this instance. + + Return value is a list of dictionaries in the following form: + + [{"_name": "", "_character_set": "", "_collate": ""}, ...] + """ + results = pgutil.query( + pgutil.DatabaseQuery.list(ignore=IGNORE_DBS_LIST), + timeout=30, + ) + # Convert results to dictionaries. + results = ( + {'_name': r[0].strip(), '_character_set': r[1], '_collate': r[2]} + for r in results + ) + # Force __iter__ of generator until marker found. + if marker is not None: + try: + item = results.next() + while item['_name'] != marker: + item = results.next() + except StopIteration: + pass + + remainder = None + if limit is not None: + remainder = results + results = itertools.islice(results, limit) + + results = tuple(results) + + next_marker = None + if remainder is not None: + try: + next_marker = remainder.next() + except StopIteration: + pass + + return results, next_marker diff --git a/trove/guestagent/datastore/postgresql/service/install.py b/trove/guestagent/datastore/postgresql/service/install.py new file mode 100644 index 0000000000..4009961421 --- /dev/null +++ b/trove/guestagent/datastore/postgresql/service/install.py @@ -0,0 +1,92 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from trove.common import cfg +from trove.common import instance +from trove.guestagent import pkg +from trove.guestagent.datastore.postgresql.service.process import PgSqlProcess +from trove.guestagent.datastore.postgresql.service.status import PgSqlAppStatus +from trove.openstack.common import log as logging +from trove.openstack.common.gettextutils import _ + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class PgSqlInstall(PgSqlProcess): + """Mixin class that provides a PgSql installer. + + This mixin has a dependency on the PgSqlProcess mixin. + """ + + def install(self, context, packages): + """Install one or more packages that postgresql needs to run. + + The packages parameter is a string representing the package names that + should be given to the system's package manager. + """ + + LOG.debug( + "{guest_id}: Beginning PgSql package installation.".format( + guest_id=CONF.guest_id + ) + ) + PgSqlAppStatus.get().begin_install() + packager = pkg.Package() + if not packager.pkg_is_installed(packages): + try: + LOG.info( + _("{guest_id}: Installing ({packages}).").format( + guest_id=CONF.guest_id, + packages=packages, + ) + ) + packager.pkg_install(packages, {}, 1000) + except (pkg.PkgAdminLockError, pkg.PkgPermissionError, + pkg.PkgPackageStateError, pkg.PkgNotFoundError, + pkg.PkgTimeout, pkg.PkgScriptletError, + pkg.PkgDownloadError, pkg.PkgSignError, + pkg.PkgBrokenError): + LOG.exception( + "{guest_id}: There was a package manager error while " + "trying to install ({packages}).".format( + guest_id=CONF.guest_id, + packages=packages, + ) + ) + PgSqlAppStatus.get().end_install_or_restart() + PgSqlAppStatus.get().set_status( + instance.ServiceStatuses.FAILED + ) + except Exception: + LOG.exception( + "{guest_id}: The package manager encountered an unknown " + "error while trying to install ({packages}).".format( + guest_id=CONF.guest_id, + packages=packages, + ) + ) + PgSqlAppStatus.get().end_install_or_restart() + PgSqlAppStatus.get().set_status( + instance.ServiceStatuses.FAILED + ) + else: + self.start_db(context) + PgSqlAppStatus.get().end_install_or_restart() + LOG.debug( + "{guest_id}: Completed package installation.".format( + guest_id=CONF.guest_id, + ) + ) diff --git a/trove/guestagent/datastore/postgresql/service/process.py b/trove/guestagent/datastore/postgresql/service/process.py new file mode 100644 index 0000000000..969eb747af --- /dev/null +++ b/trove/guestagent/datastore/postgresql/service/process.py @@ -0,0 +1,74 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from trove.common import cfg +from trove.common import utils +from trove.guestagent.common import operating_system +from trove.guestagent.datastore.postgresql.service.status import PgSqlAppStatus +from trove.openstack.common import log as logging +from trove.openstack.common.gettextutils import _ + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + +PGSQL_SERVICE_CANDIDATES = ("postgresql",) + + +class PgSqlProcess(object): + """Mixin that manages the PgSql process.""" + + def start_db(self, context): + """Start the PgSql service.""" + cmd = operating_system.service_discovery(PGSQL_SERVICE_CANDIDATES) + LOG.info( + _("{guest_id}: Starting database engine with command ({command}).") + .format( + guest_id=CONF.guest_id, + command=cmd['cmd_start'], + ) + ) + utils.execute_with_timeout( + *cmd['cmd_start'].split(), + timeout=30 + ) + + def stop_db(self, context): + """Stop the PgSql service.""" + cmd = operating_system.service_discovery(PGSQL_SERVICE_CANDIDATES) + LOG.info( + _("{guest_id}: Stopping database engine with command ({command}).") + .format( + guest_id=CONF.guest_id, + command=cmd['cmd_stop'], + ) + ) + utils.execute_with_timeout( + *cmd['cmd_stop'].split(), + timeout=30 + ) + + def restart(self, context): + """Restart the PgSql service.""" + LOG.info( + _("{guest_id}: Restarting database engine.").format( + guest_id=CONF.guest_id, + ) + ) + try: + PgSqlAppStatus.get().begin_restart() + self.stop_db(context) + self.start_db(context) + finally: + PgSqlAppStatus.get().end_install_or_restart() diff --git a/trove/guestagent/datastore/postgresql/service/root.py b/trove/guestagent/datastore/postgresql/service/root.py new file mode 100644 index 0000000000..205a6d1011 --- /dev/null +++ b/trove/guestagent/datastore/postgresql/service/root.py @@ -0,0 +1,77 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import uuid +from trove.common import cfg +from trove.guestagent.datastore.postgresql import pgutil +from trove.openstack.common import log as logging + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF +IGNORE_USERS_LIST = CONF.get(CONF.datastore_manager).ignore_users + + +class PgSqlRoot(object): + """Mixin that provides the root-enable API.""" + + def is_root_enabled(self, context): + """Return True if there is a superuser account enabled. + + This ignores the built-in superuser of postgres and the potential + system administration superuser of os_admin. + """ + results = pgutil.query( + pgutil.UserQuery.list_root(ignore=IGNORE_USERS_LIST), + timeout=30, + ) + # Reduce iter of iters to iter of single values. + results = (r[0] for r in results) + return len(tuple(results)) > 0 + + def enable_root(self, context, root_password=None): + """Create a root user or reset the root user password. + + The default superuser for PgSql is postgres, but that account is used + for administration. Instead, this method will create a new user called + root that also has superuser privileges. + + If no root_password is given then a random UUID will be used for the + superuser password. + + Return value is a dictionary in the following form: + + {"_name": "root", "_password": ""} + """ + user = { + "_name": "root", + "_password": root_password or str(uuid.uuid4()), + } + LOG.debug( + "{guest_id}: Creating root user with password {password}.".format( + guest_id=CONF.guest_id, + password=user['_password'], + ) + ) + query = pgutil.UserQuery.create( + name=user['_name'], + password=user['_password'], + ) + if self.is_root_enabled(context): + query = pgutil.UserQuery.update_password( + name=user['_name'], + password=user['_password'], + ) + pgutil.psql(query, timeout=30) + return user diff --git a/trove/guestagent/datastore/postgresql/service/status.py b/trove/guestagent/datastore/postgresql/service/status.py new file mode 100644 index 0000000000..dd3ebdabf5 --- /dev/null +++ b/trove/guestagent/datastore/postgresql/service/status.py @@ -0,0 +1,76 @@ +# Copyright (c) 2014 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +from trove.common import utils +from trove.common import exception +from trove.common import instance +from trove.guestagent.datastore import service +from trove.guestagent.datastore.postgresql import pgutil +from trove.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + +PGSQL_PID = "'/var/run/postgresql/postgresql.pid'" + + +class PgSqlAppStatus(service.BaseDbStatus): + @classmethod + def get(cls): + if not cls._instance: + cls._instance = PgSqlAppStatus() + return cls._instance + + def _get_actual_db_status(self): + """Checks the acutal PgSql process to determine status. + + Status will be one of the following: + + - RUNNING + + The process is running and responsive. + + - BLOCKED + + The process is running but unresponsive. + + - CRASHED + + The process is not running, but should be or the process + is running and should not be. + + - SHUTDOWN + + The process was gracefully shut down. + """ + + # Run a simple scalar query to make sure the process is responsive. + try: + pgutil.execute('psql', '-c', 'SELECT 1') + except utils.Timeout: + return instance.ServiceStatuses.BLOCKED + except exception.ProcessExecutionError: + try: + utils.execute_with_timeout( + "/bin/ps", "-C", "postgres", "h" + ) + except exception.ProcessExecutionError: + if os.path.exists(PGSQL_PID): + return instance.ServiceStatuses.CRASHED + return instance.ServiceStatuses.SHUTDOWN + else: + return instance.ServiceStatuses.BLOCKED + else: + return instance.ServiceStatuses.RUNNING diff --git a/trove/guestagent/datastore/postgresql/service/users.py b/trove/guestagent/datastore/postgresql/service/users.py new file mode 100644 index 0000000000..4e151a2575 --- /dev/null +++ b/trove/guestagent/datastore/postgresql/service/users.py @@ -0,0 +1,254 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools +from trove.common import cfg +from trove.guestagent.datastore.postgresql import pgutil +from trove.guestagent.datastore.postgresql.service.access import PgSqlAccess +from trove.openstack.common import log as logging +from trove.openstack.common.gettextutils import _ + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF +IGNORE_USERS_LIST = CONF.get(CONF.datastore_manager).ignore_users + + +class PgSqlUsers(PgSqlAccess): + """Mixin implementing the user CRUD API. + + This mixin has a dependency on the PgSqlAccess mixin. + """ + + def create_user(self, context, users): + """Create users and grant privileges for the specified databases. + + The users parameter is a list of dictionaries in the following form: + + {"_name": "", "_password": "", "_databases": [{"_name": ""}, ...]} + """ + for user in users: + LOG.debug( + "{guest_id}: Creating user {name} with password {password}." + .format( + guest_id=CONF.guest_id, + name=user['_name'], + password=user['_password'], + ) + ) + LOG.info( + _("{guest_id}: Creating user {name} with password {password}.") + .format( + guest_id=CONF.guest_id, + name=user['_name'], + password="", + ) + ) + pgutil.psql( + pgutil.UserQuery.create( + name=user['_name'], + password=user['_password'], + ), + timeout=30, + ) + self.grant_access( + context, + user['_name'], + None, + [d['_name'] for d in user['_databases']], + ) + + def list_users( + self, + context, + limit=None, + marker=None, + include_marker=False, + ): + """List all users on the instance along with their access permissions. + + Return value is a list of dictionaries in the following form: + + [{"_name": "", "_password": None, "_host": None, + "_databases": [{"_name": ""}, ...]}, ...] + """ + results = pgutil.query( + pgutil.UserQuery.list(ignore=IGNORE_USERS_LIST), + timeout=30, + ) + # Convert results into dictionaries. + results = ( + { + '_name': r[0].strip(), + '_password': None, + '_host': None, + '_databases': self.list_access(context, r[0], None), + } + for r in results + ) + + # Force __iter__ of generator until marker found. + if marker is not None: + try: + item = results.next() + while item['_name'] != marker: + item = results.next() + except StopIteration: + pass + + remainder = None + if limit is not None: + remainder = results + results = itertools.islice(results, limit) + + results = tuple(results) + + next_marker = None + if remainder is not None: + try: + next_marker = remainder.next() + except StopIteration: + pass + + return results, next_marker + + def delete_user(self, context, user): + """Delete the specified user. + + The user parameter is a dictionary in the following form: + + {"_name": ""} + """ + LOG.info( + _("{guest_id}: Dropping user {name}.").format( + guest_id=CONF.guest_id, + name=user['_name'], + ) + ) + pgutil.psql( + pgutil.UserQuery.drop(name=user['_name']), + timeout=30, + ) + + def get_user(self, context, username, hostname): + """Return a single user matching the criteria. + + The username and hostname parameter are strings. + + The return value is a dictionary in the following form: + + {"_name": "", "_host": None, "_password": None, + "_databases": [{"_name": ""}, ...]} + + Where "_databases" is a list of databases the user has access to. + """ + results = pgutil.query( + pgutil.UserQuery.get(name=username), + timeout=30, + ) + results = tuple(results) + if len(results) < 1: + return None + + return { + "_name": results[0][0], + "_host": None, + "_password": None, + "_databases": self.list_access(context, username, None), + } + + def change_passwords(self, context, users): + """Change the passwords of one or more existing users. + + The users parameter is a list of dictionaries in the following form: + + {"name": "", "password": ""} + """ + for user in users: + LOG.debug( + "{guest_id}: Changing password for {user} to {password}." + .format( + guest_id=CONF.guest_id, + user=user['name'], + password=user['password'], + ) + ) + LOG.info( + _("{guest_id}: Changing password for {user} to {password}.") + .format( + guest_id=CONF.guest_id, + user=user['name'], + password="", + ) + ) + pgutil.psql( + pgutil.UserQuery.update_password( + user=user['name'], + password=user['password'], + ), + timeout=30, + ) + + def update_attributes(self, context, username, hostname, user_attrs): + """Change the attributes of one existing user. + + The username and hostname parameters are strings. + The user_attrs parameter is a dictionary in the following form: + + {"password": "", "name": ""} + + Each key/value pair in user_attrs is optional. + """ + if user_attrs.get('password') is not None: + self.change_passwords( + context, + ( + { + "name": username, + "password": user_attrs['password'], + }, + ), + ) + + if user_attrs.get('name') is not None: + access = self.list_access(context, username, None) + LOG.info( + _("{guest_id}: Changing username for {old} to {new}.").format( + guest_id=CONF.guest_id, + old=username, + new=user_attrs['name'], + ) + ) + pgutil.psql( + pgutil.psql.UserQuery.update_name( + old=username, + new=user_attrs['name'], + ), + timeout=30, + ) + # Regrant all previous access after the name change. + LOG.info( + _("{guest_id}: Regranting permissions from {old} to {new}.") + .format( + guest_id=CONF.guest_id, + old=username, + new=user_attrs['name'], + ) + ) + self.grant_access( + context, + username=user_attrs['name'], + hostname=None, + databases=(db['_name'] for db in access) + ) diff --git a/trove/guestagent/dbaas.py b/trove/guestagent/dbaas.py index 48c2aedeba..ee008ae15b 100644 --- a/trove/guestagent/dbaas.py +++ b/trove/guestagent/dbaas.py @@ -39,6 +39,7 @@ defaults = { 'cassandra': 'trove.guestagent.datastore.cassandra.manager.Manager', 'couchbase': 'trove.guestagent.datastore.couchbase.manager.Manager', 'mongodb': 'trove.guestagent.datastore.mongodb.manager.Manager', + 'postgresql': 'trove.guestagent.datastore.postgresql.manager.Manager', } CONF = cfg.CONF diff --git a/trove/guestagent/pkg.py b/trove/guestagent/pkg.py index 7906979112..2761d82538 100644 --- a/trove/guestagent/pkg.py +++ b/trove/guestagent/pkg.py @@ -20,7 +20,7 @@ import commands import re import subprocess from tempfile import NamedTemporaryFile - +import os import pexpect import six diff --git a/trove/guestagent/strategies/backup/postgresql_impl.py b/trove/guestagent/strategies/backup/postgresql_impl.py new file mode 100644 index 0000000000..7442cc4e0c --- /dev/null +++ b/trove/guestagent/strategies/backup/postgresql_impl.py @@ -0,0 +1,29 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from trove.guestagent.strategies.backup import base +from trove.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + + +class PgDump(base.BackupRunner): + """Implementation of Backup Strategy for pg_dump.""" + __strategy_name__ = 'pg_dump' + + @property + def cmd(self): + cmd = 'sudo -u postgres pg_dumpall ' + return cmd + self.zip_cmd + self.encrypt_cmd diff --git a/trove/guestagent/strategies/restore/postgresql_impl.py b/trove/guestagent/strategies/restore/postgresql_impl.py new file mode 100644 index 0000000000..8e2383899d --- /dev/null +++ b/trove/guestagent/strategies/restore/postgresql_impl.py @@ -0,0 +1,25 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from trove.guestagent.strategies.restore import base +from trove.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + + +class PgDump(base.RestoreRunner): + """Implementation of Restore Strategy for pg_dump.""" + __strategy_name__ = 'pg_dump' + base_restore_cmd = 'sudo -u postgres psql ' diff --git a/trove/templates/postgresql/config.template b/trove/templates/postgresql/config.template new file mode 100644 index 0000000000..e37faf2b3f --- /dev/null +++ b/trove/templates/postgresql/config.template @@ -0,0 +1,24 @@ +data_directory = '/var/lib/postgresql/{{datastore['version']}}/main' + +hba_file = '/etc/postgresql/{{datastore['version']}}/main/pg_hba.conf' + +ident_file = '/etc/postgresql/{{datastore['version']}}/main/pg_ident.conf' + +external_pid_file = '/var/run/postgresql/postgresql.pid' + +listen_addresses = '*' + +port = 5432 + +max_connections = 100 + +shared_buffers = 24MB + +log_line_prefix = '%t ' + +lc_messages = 'en_US.UTF-8' +lc_monetary = 'en_US.UTF-8' +lc_numeric = 'en_US.UTF-8' +lc_time = 'en_US.UTF-8' + +default_text_search_config = 'pg_catalog.english' diff --git a/trove/templates/postgresql/override.config.template b/trove/templates/postgresql/override.config.template new file mode 100644 index 0000000000..e69de29bb2 diff --git a/trove/tests/int_tests.py b/trove/tests/int_tests.py index 56f1add159..f96747a587 100644 --- a/trove/tests/int_tests.py +++ b/trove/tests/int_tests.py @@ -91,5 +91,5 @@ datastore_group = [ versions.GROUP, instances.GROUP_START_SIMPLE, ] -proboscis.register(groups=["cassandra", "couchbase", "mongodb"], +proboscis.register(groups=["cassandra", "couchbase", "mongodb", "postgresql"], depends_on_groups=datastore_group)