openstack
/
trove
Code Issues Proposed changes

Merge "Add PostgreSQL support"

This commit is contained in:
Jenkins 2014-09-04 00:23:29 +00:00 committed by Gerrit Code Review
parent 74b436e1d2 4fdd1b5dea
commit a679d2626a
20 changed files with 1359 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
trove/common/cfg.py
View File

@ -524,6 +524,41 @@ mongodb_opts = [
'logic.'),
]
# PostgreSQL
postgresql_group = cfg.OptGroup(
'postgresql', title='PostgreSQL options',
help="Oslo option group for the PostgreSQL datastore.")
postgresql_opts = [
cfg.ListOpt('tcp_ports', default=["5432"],
help='List of TCP ports and/or port ranges to open'
' in the security group (only applicable '
'if trove_security_groups_support is True).'),
cfg.ListOpt('udp_ports', default=[],
help='List of UPD ports and/or port ranges to open'
' in the security group (only applicable '
'if trove_security_groups_support is True).'),
cfg.StrOpt('backup_strategy', default='PgDump',
help='Default strategy to perform backups.'),
cfg.StrOpt('mount_point', default='/var/lib/postgresql',
help="Filesystem path for mounting "
"volumes if volume support is enabled."),
cfg.BoolOpt('root_on_create', default=False,
help='Enable the automatic creation of the root user for the '
'service during instance-create. The generated password for '
'the root user is immediately returned in the response of '
"instance-create as the 'password' field."),
cfg.StrOpt('backup_namespace',
default='trove.guestagent.strategies.backup.postgresql_impl'),
cfg.StrOpt('restore_namespace',
default='trove.guestagent.strategies.restore.postgresql_impl'),
cfg.BoolOpt('volume_support',
default=True,
help='Whether to provision a cinder volume for datadir.'),
cfg.StrOpt('device_path', default='/dev/vdb'),
cfg.ListOpt('ignore_users', default=['os_admin', 'postgres', 'root']),
cfg.ListOpt('ignore_dbs', default=['postgres']),
]
CONF = cfg.CONF
CONF.register_opts(path_opts)
@ -542,6 +577,7 @@ CONF.register_opts(redis_opts, redis_group)
CONF.register_opts(cassandra_opts, cassandra_group)
CONF.register_opts(couchbase_opts, couchbase_group)
CONF.register_opts(mongodb_opts, mongodb_group)
CONF.register_opts(postgresql_opts, postgresql_group)
def custom_parser(parsername, parser):

0
trove/guestagent/datastore/postgresql/__init__.py Normal file
View File

98
trove/guestagent/datastore/postgresql/manager.py Normal file
View File

@ -0,0 +1,98 @@
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from trove.common import cfg
from trove.guestagent import dbaas
from trove.guestagent import backup
from trove.guestagent import volume
from .service.config import PgSqlConfig
from .service.database import PgSqlDatabase
from .service.install import PgSqlInstall
from .service.root import PgSqlRoot
from .service.users import PgSqlUsers
from .service.status import PgSqlAppStatus
from trove.openstack.common import log as logging
from trove.openstack.common import periodic_task
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class Manager(
periodic_task.PeriodicTasks,
PgSqlUsers,
PgSqlDatabase,
PgSqlRoot,
PgSqlConfig,
PgSqlInstall,
):
def __init__(self, *args, **kwargs):
super(Manager, self).__init__(*args, **kwargs)
@periodic_task.periodic_task(ticks_between_runs=3)
def update_status(self, context):
PgSqlAppStatus.get().update()
def prepare(
self,
context,
packages,
databases,
memory_mb,
users,
device_path=None,
mount_point=None,
backup_info=None,
config_contents=None,
root_password=None,
overrides=None,
):
self.install(context, packages)
PgSqlAppStatus.get().begin_restart()
self.stop_db(context)
if device_path:
device = volume.VolumeDevice(device_path)
device.format()
if os.path.exists(mount_point):
if not backup_info:
device.migrate_data(mount_point)
device.mount(mount_point)
self.reset_configuration(context, config_contents)
self.set_db_to_listen(context)
self.start_db(context)
if backup_info:
backup.restore(context, backup_info, '/tmp')
if root_password and not backup_info:
self.enable_root(context, root_password)
PgSqlAppStatus.get().end_install_or_restart()
if databases:
self.create_database(context, databases)
if users:
self.create_user(context, users)
def get_filesystem_stats(self, context, fs_path):
return dbaas.get_filesystem_volume_stats(fs_path)
def create_backup(self, context, backup_info):
backup.backup(context, backup_info)

229
trove/guestagent/datastore/postgresql/pgutil.py Normal file
View File

@ -0,0 +1,229 @@
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import tempfile
import uuid
from trove.common import utils
from trove.openstack.common import log as logging
LOG = logging.getLogger(__name__)
def execute(*command, **kwargs):
"""Execute a command as the 'postgres' user."""
LOG.debug('Running as postgres: {0}'.format(command))
return utils.execute_with_timeout(
"sudo", "-u", "postgres", *command, **kwargs
)
def result(filename):
"""A generator representing the results of a query.
This generator produces result records of a query by iterating over a
CSV file created by the query. When the file is out of records it is
removed.
The purpose behind this abstraction is to provide a record set interface
with minimal memory consumption without requiring an active DB connection.
This makes it possible to iterate over any sized record set without
allocating memory for the entire record set and without using a DB cursor.
Each row is returned as an iterable of column values. The order of these
values is determined by the query.
"""
utils.execute_with_timeout(
'sudo', 'chmod', '777', filename,
)
with open(filename, 'r+') as file_handle:
for line in file_handle:
if line != "":
yield line.split(',')
execute(
"rm", "{filename}".format(filename=filename),
)
raise StopIteration()
def psql(statement, timeout=30):
"""Execute a statement using the psql client."""
LOG.debug('Sending to local db: {0}'.format(statement))
return execute('psql', '-c', statement, timeout=timeout)
def query(statement, timeout=30):
"""Execute a pgsql query and get a generator of results.
This method will pipe a CSV format of the query results into a temporary
file. The return value is a generator object that feeds from this file.
"""
filename = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
LOG.debug('Querying: {0}'.format(statement))
psql(
"Copy ({statement}) To '{filename}' With CSV".format(
statement=statement,
filename=filename,
),
timeout=timeout,
)
return result(filename)
class DatabaseQuery(object):
@classmethod
def list(cls, ignore=()):
"""Query to list all databases."""
statement = (
"SELECT datname, pg_encoding_to_char(encoding), "
"datcollate FROM pg_database "
"WHERE datistemplate = false"
)
for name in ignore:
statement += " AND datname != '{name}'".format(name=name)
return statement
@classmethod
def create(cls, name, encoding=None, collation=None):
"""Query to create a database."""
statement = "CREATE DATABASE {name}".format(name=name)
if encoding is not None:
statement += " ENCODING = {encoding}".format(
encoding=encoding,
)
if collation is not None:
statement += " LC_COLLATE = {collation}".format(
collation=collation,
)
return statement
@classmethod
def drop(cls, name):
"""Query to drop a database."""
return "DROP DATABASE IF EXISTS {name}".format(name=name)
class UserQuery(object):
@classmethod
def list(cls, ignore=()):
"""Query to list all users."""
statement = "SELECT usename FROM pg_catalog.pg_user"
if ignore:
# User a simple tautology so all clauses can be AND'ed without
# crazy special logic.
statement += " WHERE 1=1"
for name in ignore:
statement += " AND usename != '{name}'".format(name=name)
return statement
@classmethod
def list_root(cls, ignore=()):
"""Query to list all superuser accounts."""
statement = (
"SELECT usename FROM pg_catalog.pg_user WHERE usesuper = true"
)
for name in ignore:
statement += " AND usename != '{name}'".format(name=name)
return statement
@classmethod
def get(cls, name):
"""Query to get a single user."""
return (
"SELECT usename FROM pg_catalog.pg_user "
"WHERE usename = '{name}'".format(name=name)
)
@classmethod
def create(cls, name, password):
"""Query to create a user with a password."""
return "CREATE USER {name} WITH PASSWORD '{password}'".format(
name=name,
password=password,
)
@classmethod
def update_password(cls, name, password):
"""Query to update the password for a user."""
return "ALTER USER {name} WITH PASSWORD '{password}'".format(
name=name,
password=password,
)
@classmethod
def update_name(cls, old, new):
"""Query to update the name of a user."""
return "ALTER USER {old} RENAME TO {new}".format(old=old, new=new)
@classmethod
def drop(cls, name):
"""Query to drop a user."""
return "DROP USER {name}".format(name=name)
class AccessQuery(object):
@classmethod
def list(cls, user):
"""Query to list grants for a user."""
return (
"SELECT datname "
"FROM pg_database "
"WHERE datistemplate = false "
"AND 'user {user}=CTc' = ANY (datacl)".format(user=user)
)
@classmethod
def grant(cls, user, database):
"""Query to grant user access to a database."""
return "GRANT ALL ON DATABASE {database} TO {user}".format(
database=database,
user=user,
)
@classmethod
def revoke(cls, user, database):
"""Query to revoke user access to a database."""
return "REVOKE ALL ON DATABASE {database} FROM {user}".format(
database=database,
user=user,
)

0
trove/guestagent/datastore/postgresql/service/__init__.py Normal file
View File

94
trove/guestagent/datastore/postgresql/service/access.py Normal file
View File

@ -0,0 +1,94 @@
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from trove.common import cfg
from trove.guestagent.datastore.postgresql import pgutil
from trove.openstack.common import log as logging
from trove.openstack.common.gettextutils import _
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class PgSqlAccess(object):
"""Mixin implementing the user-access API calls."""
def grant_access(self, context, username, hostname, databases):
"""Give a user permission to use a given database.
The username and hostname parameters are strings.
The databases parameter is a list of strings representing the names of
the databases to grant permission on.
"""
for database in databases:
LOG.info(
_("{guest_id}: Granting user ({user}) access to database "
"({database}).").format(
guest_id=CONF.guest_id,
user=username,
database=database,
)
)
pgutil.psql(
pgutil.AccessQuery.grant(
user=username,
database=database,
),
timeout=30,
)
def revoke_access(self, context, username, hostname, database):
"""Revoke a user's permission to use a given database.
The username and hostname parameters are strings.
The database parameter is a string representing the name of the
database.
"""
LOG.info(
_("{guest_id}: Revoking user ({user}) access to database"
"({database}).").format(
guest_id=CONF.guest_id,
user=username,
database=database,
)
)
pgutil.psql(
pgutil.AccessQuery.revoke(
user=username,
database=database,
),
timeout=30,
)
def list_access(self, context, username, hostname):
"""List database for which the given user as access.
The username and hostname parameters are strings.
Return value is a list of dictionaries in the following form:
[{"_name": "", "_collate": None, "_character_set": None}, ...]
"""
results = pgutil.query(
pgutil.AccessQuery.list(user=username),
timeout=30,
)
# Convert to dictionaries.
results = (
{'_name': r[0].strip(), '_collate': None, '_character_set': None}
for r in results
)
return tuple(results)

127
trove/guestagent/datastore/postgresql/service/config.py Normal file
View File

@ -0,0 +1,127 @@
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
from trove.common import cfg
from trove.common import utils
from trove.guestagent.datastore.postgresql import pgutil
from trove.guestagent.datastore.postgresql.service.process import PgSqlProcess
from trove.guestagent.datastore.postgresql.service.status import PgSqlAppStatus
from trove.openstack.common import log as logging
from trove.openstack.common.gettextutils import _
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
PGSQL_CONFIG = "/etc/postgresql/{version}/main/postgresql.conf"
PGSQL_HBA_CONFIG = "/etc/postgresql/{version}/main/pg_hba.conf"
class PgSqlConfig(PgSqlProcess):
"""Mixin that implements the config API.
This mixin has a dependency on the PgSqlProcess mixin.
"""
def _get_psql_version(self):
"""Poll PgSql for the version number.
Return value is a string representing the version number.
"""
LOG.debug(
"{guest_id}: Polling for postgresql version.".format(
guest_id=CONF.guest_id,
)
)
out, err = pgutil.execute('psql', '--version', timeout=30)
pattern = re.compile('\d\.\d')
return pattern.search(out).group(0)
def reset_configuration(self, context, configuration):
"""Reset the PgSql configuration file to the one given.
The configuration parameter is a string containing the full
configuration file that should be used.
"""
config_location = PGSQL_CONFIG.format(
version=self._get_psql_version(),
)
LOG.debug(
"{guest_id}: Writing configuration file to /tmp/pgsql_config."
.format(
guest_id=CONF.guest_id,
)
)
with open('/tmp/pgsql_config', 'w+') as config_file:
config_file.write(configuration)
utils.execute_with_timeout(
'sudo', 'chown', 'postgres', '/tmp/pgsql_config',
timeout=30,
)
utils.execute_with_timeout(
'sudo', 'mv', '/tmp/pgsql_config', config_location,
timeout=30,
)
def set_db_to_listen(self, context):
"""Allow remote connections with encrypted passwords."""
# Using cat to read file due to read permissions issues.
out, err = utils.execute_with_timeout(
'sudo', 'cat',
PGSQL_HBA_CONFIG.format(
version=self._get_psql_version(),
),
timeout=30,
)
LOG.debug(
"{guest_id}: Writing hba file to /tmp/pgsql_hba_config.".format(
guest_id=CONF.guest_id,
)
)
with open('/tmp/pgsql_hba_config', 'w+') as config_file:
config_file.write(out)
config_file.write("host all all 0.0.0.0/0 md5\n")
utils.execute_with_timeout(
'sudo', 'chown', 'postgres', '/tmp/pgsql_hba_config',
timeout=30,
)
utils.execute_with_timeout(
'sudo', 'mv', '/tmp/pgsql_hba_config',
PGSQL_HBA_CONFIG.format(
version=self._get_psql_version(),
),
timeout=30,
)
def start_db_with_conf_changes(self, context, config_contents):
"""Restarts the PgSql instance with a new configuration."""
LOG.info(
_("{guest_id}: Going into restart mode for config file changes.")
.format(
guest_id=CONF.guest_id,
)
)
PgSqlAppStatus.get().begin_restart()
self.stop_db(context)
self.reset_configuration(context, config_contents)
self.start_db(context)
LOG.info(
_("{guest_id}: Ending restart mode for config file changes.")
.format(
guest_id=CONF.guest_id,
)
)
PgSqlAppStatus.get().end_install_or_restart()

121
trove/guestagent/datastore/postgresql/service/database.py Normal file
View File

@ -0,0 +1,121 @@
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
from trove.common import cfg
from trove.guestagent.datastore.postgresql import pgutil
from trove.openstack.common import log as logging
from trove.openstack.common.gettextutils import _
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
IGNORE_DBS_LIST = CONF.get(CONF.datastore_manager).ignore_dbs
class PgSqlDatabase(object):
def create_database(self, context, databases):
"""Create the list of specified databases.
The databases parameter is a list of dictionaries in the following
form:
{"_name": "", "_character_set": "", "_collate": ""}
Encoding and collation values are validated in
trove.guestagent.db.models.
"""
for database in databases:
encoding = database.get('_character_set')
collate = database.get('_collate')
LOG.info(
_("{guest_id}: Creating database {name}.").format(
guest_id=CONF.guest_id,
name=database['_name'],
)
)
pgutil.psql(
pgutil.DatabaseQuery.create(
name=database['_name'],
encoding=encoding,
collation=collate,
),
timeout=30,
)
def delete_database(self, context, database):
"""Delete the specified database.
The database parameter is a dictionary in the following form:
{"_name": ""}
"""
LOG.info(
_("{guest_id}: Dropping database {name}.").format(
guest_id=CONF.guest_id,
name=database['_name'],
)
)
pgutil.psql(
pgutil.DatabaseQuery.drop(name=database['_name']),
timeout=30,
)
def list_databases(
self,
context,
limit=None,
marker=None,
include_marker=False,
):
"""List databases created on this instance.
Return value is a list of dictionaries in the following form:
[{"_name": "", "_character_set": "", "_collate": ""}, ...]
"""
results = pgutil.query(
pgutil.DatabaseQuery.list(ignore=IGNORE_DBS_LIST),
timeout=30,
)
# Convert results to dictionaries.
results = (
{'_name': r[0].strip(), '_character_set': r[1], '_collate': r[2]}
for r in results
)
# Force __iter__ of generator until marker found.
if marker is not None:
try:
item = results.next()
while item['_name'] != marker:
item = results.next()
except StopIteration:
pass
remainder = None
if limit is not None:
remainder = results
results = itertools.islice(results, limit)
results = tuple(results)
next_marker = None
if remainder is not None:
try:
next_marker = remainder.next()
except StopIteration:
pass
return results, next_marker

92
trove/guestagent/datastore/postgresql/service/install.py Normal file
View File

@ -0,0 +1,92 @@
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from trove.common import cfg
from trove.common import instance
from trove.guestagent import pkg
from trove.guestagent.datastore.postgresql.service.process import PgSqlProcess
from trove.guestagent.datastore.postgresql.service.status import PgSqlAppStatus
from trove.openstack.common import log as logging
from trove.openstack.common.gettextutils import _
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class PgSqlInstall(PgSqlProcess):
"""Mixin class that provides a PgSql installer.
This mixin has a dependency on the PgSqlProcess mixin.
"""
def install(self, context, packages):
"""Install one or more packages that postgresql needs to run.
The packages parameter is a string representing the package names that
should be given to the system's package manager.
"""
LOG.debug(
"{guest_id}: Beginning PgSql package installation.".format(
guest_id=CONF.guest_id
)
)
PgSqlAppStatus.get().begin_install()
packager = pkg.Package()
if not packager.pkg_is_installed(packages):
try:
LOG.info(
_("{guest_id}: Installing ({packages}).").format(
guest_id=CONF.guest_id,
packages=packages,
)
)
packager.pkg_install(packages, {}, 1000)
except (pkg.PkgAdminLockError, pkg.PkgPermissionError,
pkg.PkgPackageStateError, pkg.PkgNotFoundError,
pkg.PkgTimeout, pkg.PkgScriptletError,
pkg.PkgDownloadError, pkg.PkgSignError,
pkg.PkgBrokenError):
LOG.exception(
"{guest_id}: There was a package manager error while "
"trying to install ({packages}).".format(
guest_id=CONF.guest_id,
packages=packages,
)
)
PgSqlAppStatus.get().end_install_or_restart()
PgSqlAppStatus.get().set_status(
instance.ServiceStatuses.FAILED
)
except Exception:
LOG.exception(
"{guest_id}: The package manager encountered an unknown "
"error while trying to install ({packages}).".format(
guest_id=CONF.guest_id,
packages=packages,
)
)
PgSqlAppStatus.get().end_install_or_restart()
PgSqlAppStatus.get().set_status(
instance.ServiceStatuses.FAILED
)
else:
self.start_db(context)
PgSqlAppStatus.get().end_install_or_restart()
LOG.debug(
"{guest_id}: Completed package installation.".format(
guest_id=CONF.guest_id,
)
)

74
trove/guestagent/datastore/postgresql/service/process.py Normal file
View File

@ -0,0 +1,74 @@
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from trove.common import cfg
from trove.common import utils
from trove.guestagent.common import operating_system
from trove.guestagent.datastore.postgresql.service.status import PgSqlAppStatus
from trove.openstack.common import log as logging
from trove.openstack.common.gettextutils import _
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
PGSQL_SERVICE_CANDIDATES = ("postgresql",)
class PgSqlProcess(object):
"""Mixin that manages the PgSql process."""
def start_db(self, context):
"""Start the PgSql service."""
cmd = operating_system.service_discovery(PGSQL_SERVICE_CANDIDATES)
LOG.info(
_("{guest_id}: Starting database engine with command ({command}).")
.format(
guest_id=CONF.guest_id,
command=cmd['cmd_start'],
)
)
utils.execute_with_timeout(
*cmd['cmd_start'].split(),
timeout=30
)
def stop_db(self, context):
"""Stop the PgSql service."""
cmd = operating_system.service_discovery(PGSQL_SERVICE_CANDIDATES)
LOG.info(
_("{guest_id}: Stopping database engine with command ({command}).")
.format(
guest_id=CONF.guest_id,
command=cmd['cmd_stop'],
)
)
utils.execute_with_timeout(
*cmd['cmd_stop'].split(),
timeout=30
)
def restart(self, context):
"""Restart the PgSql service."""
LOG.info(
_("{guest_id}: Restarting database engine.").format(
guest_id=CONF.guest_id,
)
)
try:
PgSqlAppStatus.get().begin_restart()
self.stop_db(context)
self.start_db(context)
finally:
PgSqlAppStatus.get().end_install_or_restart()

77
trove/guestagent/datastore/postgresql/service/root.py Normal file
View File

@ -0,0 +1,77 @@
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import uuid
from trove.common import cfg
from trove.guestagent.datastore.postgresql import pgutil
from trove.openstack.common import log as logging
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
IGNORE_USERS_LIST = CONF.get(CONF.datastore_manager).ignore_users
class PgSqlRoot(object):
"""Mixin that provides the root-enable API."""
def is_root_enabled(self, context):
"""Return True if there is a superuser account enabled.
This ignores the built-in superuser of postgres and the potential
system administration superuser of os_admin.
"""
results = pgutil.query(
pgutil.UserQuery.list_root(ignore=IGNORE_USERS_LIST),
timeout=30,
)
# Reduce iter of iters to iter of single values.
results = (r[0] for r in results)
return len(tuple(results)) > 0
def enable_root(self, context, root_password=None):
"""Create a root user or reset the root user password.
The default superuser for PgSql is postgres, but that account is used
for administration. Instead, this method will create a new user called
root that also has superuser privileges.
If no root_password is given then a random UUID will be used for the
superuser password.
Return value is a dictionary in the following form:
{"_name": "root", "_password": ""}
"""
user = {
"_name": "root",
"_password": root_password or str(uuid.uuid4()),
}
LOG.debug(
"{guest_id}: Creating root user with password {password}.".format(
guest_id=CONF.guest_id,
password=user['_password'],
)
)
query = pgutil.UserQuery.create(
name=user['_name'],
password=user['_password'],
)
if self.is_root_enabled(context):
query = pgutil.UserQuery.update_password(
name=user['_name'],
password=user['_password'],
)
pgutil.psql(query, timeout=30)
return user

76
trove/guestagent/datastore/postgresql/service/status.py Normal file
View File

@ -0,0 +1,76 @@
# Copyright (c) 2014 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from trove.common import utils
from trove.common import exception
from trove.common import instance
from trove.guestagent.datastore import service
from trove.guestagent.datastore.postgresql import pgutil
from trove.openstack.common import log as logging
LOG = logging.getLogger(__name__)
PGSQL_PID = "'/var/run/postgresql/postgresql.pid'"
class PgSqlAppStatus(service.BaseDbStatus):
@classmethod
def get(cls):
if not cls._instance:
cls._instance = PgSqlAppStatus()
return cls._instance
def _get_actual_db_status(self):
"""Checks the acutal PgSql process to determine status.
Status will be one of the following:
- RUNNING
The process is running and responsive.
- BLOCKED
The process is running but unresponsive.
- CRASHED
The process is not running, but should be or the process
is running and should not be.
- SHUTDOWN
The process was gracefully shut down.
"""
# Run a simple scalar query to make sure the process is responsive.
try:
pgutil.execute('psql', '-c', 'SELECT 1')
except utils.Timeout:
return instance.ServiceStatuses.BLOCKED
except exception.ProcessExecutionError:
try:
utils.execute_with_timeout(
"/bin/ps", "-C", "postgres", "h"
)
except exception.ProcessExecutionError:
if os.path.exists(PGSQL_PID):
return instance.ServiceStatuses.CRASHED
return instance.ServiceStatuses.SHUTDOWN
else:
return instance.ServiceStatuses.BLOCKED
else:
return instance.ServiceStatuses.RUNNING

254
trove/guestagent/datastore/postgresql/service/users.py Normal file
View File

@ -0,0 +1,254 @@
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
from trove.common import cfg
from trove.guestagent.datastore.postgresql import pgutil
from trove.guestagent.datastore.postgresql.service.access import PgSqlAccess
from trove.openstack.common import log as logging
from trove.openstack.common.gettextutils import _
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
IGNORE_USERS_LIST = CONF.get(CONF.datastore_manager).ignore_users
class PgSqlUsers(PgSqlAccess):
"""Mixin implementing the user CRUD API.
This mixin has a dependency on the PgSqlAccess mixin.
"""
def create_user(self, context, users):
"""Create users and grant privileges for the specified databases.
The users parameter is a list of dictionaries in the following form:
{"_name": "", "_password": "", "_databases": [{"_name": ""}, ...]}
"""
for user in users:
LOG.debug(
"{guest_id}: Creating user {name} with password {password}."
.format(
guest_id=CONF.guest_id,
name=user['_name'],
password=user['_password'],
)
)
LOG.info(
_("{guest_id}: Creating user {name} with password {password}.")
.format(
guest_id=CONF.guest_id,
name=user['_name'],
password="<SANITIZED>",
)
)
pgutil.psql(
pgutil.UserQuery.create(
name=user['_name'],
password=user['_password'],
),
timeout=30,
)
self.grant_access(
context,
user['_name'],
None,
[d['_name'] for d in user['_databases']],
)
def list_users(
self,
context,
limit=None,
marker=None,
include_marker=False,
):
"""List all users on the instance along with their access permissions.
Return value is a list of dictionaries in the following form:
[{"_name": "", "_password": None, "_host": None,
"_databases": [{"_name": ""}, ...]}, ...]
"""
results = pgutil.query(
pgutil.UserQuery.list(ignore=IGNORE_USERS_LIST),
timeout=30,
)
# Convert results into dictionaries.
results = (
{
'_name': r[0].strip(),
'_password': None,
'_host': None,
'_databases': self.list_access(context, r[0], None),
}
for r in results
)
# Force __iter__ of generator until marker found.
if marker is not None:
try:
item = results.next()
while item['_name'] != marker:
item = results.next()
except StopIteration:
pass
remainder = None
if limit is not None:
remainder = results
results = itertools.islice(results, limit)
results = tuple(results)
next_marker = None
if remainder is not None:
try:
next_marker = remainder.next()
except StopIteration:
pass
return results, next_marker
def delete_user(self, context, user):
"""Delete the specified user.
The user parameter is a dictionary in the following form:
{"_name": ""}
"""
LOG.info(
_("{guest_id}: Dropping user {name}.").format(
guest_id=CONF.guest_id,
name=user['_name'],
)
)
pgutil.psql(
pgutil.UserQuery.drop(name=user['_name']),
timeout=30,
)
def get_user(self, context, username, hostname):
"""Return a single user matching the criteria.
The username and hostname parameter are strings.
The return value is a dictionary in the following form:
{"_name": "", "_host": None, "_password": None,
"_databases": [{"_name": ""}, ...]}
Where "_databases" is a list of databases the user has access to.
"""
results = pgutil.query(
pgutil.UserQuery.get(name=username),
timeout=30,
)
results = tuple(results)
if len(results) < 1:
return None
return {
"_name": results[0][0],
"_host": None,
"_password": None,
"_databases": self.list_access(context, username, None),
}
def change_passwords(self, context, users):
"""Change the passwords of one or more existing users.
The users parameter is a list of dictionaries in the following form:
{"name": "", "password": ""}
"""
for user in users:
LOG.debug(
"{guest_id}: Changing password for {user} to {password}."
.format(
guest_id=CONF.guest_id,
user=user['name'],
password=user['password'],
)
)
LOG.info(
_("{guest_id}: Changing password for {user} to {password}.")
.format(
guest_id=CONF.guest_id,
user=user['name'],
password="<SANITIZED>",
)
)
pgutil.psql(
pgutil.UserQuery.update_password(
user=user['name'],
password=user['password'],
),
timeout=30,
)
def update_attributes(self, context, username, hostname, user_attrs):
"""Change the attributes of one existing user.
The username and hostname parameters are strings.
The user_attrs parameter is a dictionary in the following form:
{"password": "", "name": ""}
Each key/value pair in user_attrs is optional.
"""
if user_attrs.get('password') is not None:
self.change_passwords(
context,
(
{
"name": username,
"password": user_attrs['password'],
},
),
)
if user_attrs.get('name') is not None:
access = self.list_access(context, username, None)
LOG.info(
_("{guest_id}: Changing username for {old} to {new}.").format(
guest_id=CONF.guest_id,
old=username,
new=user_attrs['name'],
)
)
pgutil.psql(
pgutil.psql.UserQuery.update_name(
old=username,
new=user_attrs['name'],
),
timeout=30,
)
# Regrant all previous access after the name change.
LOG.info(
_("{guest_id}: Regranting permissions from {old} to {new}.")
.format(
guest_id=CONF.guest_id,
old=username,
new=user_attrs['name'],
)
)
self.grant_access(
context,
username=user_attrs['name'],
hostname=None,
databases=(db['_name'] for db in access)
)

1
trove/guestagent/dbaas.py
View File

@ -39,6 +39,7 @@ defaults = {
'cassandra': 'trove.guestagent.datastore.cassandra.manager.Manager',
'couchbase': 'trove.guestagent.datastore.couchbase.manager.Manager',
'mongodb': 'trove.guestagent.datastore.mongodb.manager.Manager',
'postgresql': 'trove.guestagent.datastore.postgresql.manager.Manager',
}
CONF = cfg.CONF

2
trove/guestagent/pkg.py
View File

@ -20,7 +20,7 @@ import commands
import re
import subprocess
from tempfile import NamedTemporaryFile
import os
import pexpect
import six

29
trove/guestagent/strategies/backup/postgresql_impl.py Normal file
View File

@ -0,0 +1,29 @@
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from trove.guestagent.strategies.backup import base
from trove.openstack.common import log as logging
LOG = logging.getLogger(__name__)
class PgDump(base.BackupRunner):
"""Implementation of Backup Strategy for pg_dump."""
__strategy_name__ = 'pg_dump'
@property
def cmd(self):
cmd = 'sudo -u postgres pg_dumpall '
return cmd + self.zip_cmd + self.encrypt_cmd

25
trove/guestagent/strategies/restore/postgresql_impl.py Normal file
View File

@ -0,0 +1,25 @@
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from trove.guestagent.strategies.restore import base
from trove.openstack.common import log as logging
LOG = logging.getLogger(__name__)
class PgDump(base.RestoreRunner):
"""Implementation of Restore Strategy for pg_dump."""
__strategy_name__ = 'pg_dump'
base_restore_cmd = 'sudo -u postgres psql '

24
trove/templates/postgresql/config.template Normal file
View File

@ -0,0 +1,24 @@
data_directory = '/var/lib/postgresql/{{datastore['version']}}/main'
hba_file = '/etc/postgresql/{{datastore['version']}}/main/pg_hba.conf'
ident_file = '/etc/postgresql/{{datastore['version']}}/main/pg_ident.conf'
external_pid_file = '/var/run/postgresql/postgresql.pid'
listen_addresses = '*'
port = 5432
max_connections = 100
shared_buffers = 24MB
log_line_prefix = '%t '
lc_messages = 'en_US.UTF-8'
lc_monetary = 'en_US.UTF-8'
lc_numeric = 'en_US.UTF-8'
lc_time = 'en_US.UTF-8'
default_text_search_config = 'pg_catalog.english'

0
trove/templates/postgresql/override.config.template Normal file
View File

2
trove/tests/int_tests.py
View File

@ -91,5 +91,5 @@ datastore_group = [
versions.GROUP,
instances.GROUP_START_SIMPLE,
]
proboscis.register(groups=["cassandra", "couchbase", "mongodb"],
proboscis.register(groups=["cassandra", "couchbase", "mongodb", "postgresql"],
depends_on_groups=datastore_group)