Datastore containerization

Significant changes:

* Using docker image to install datastore.
* Datastore image is common to different datastores.
* Using backup docker image to do backup and restore.
* Support MariaDB replication
* Set most of the functional jobs as non-voting as nested
  virtualization is not supported in CI.

Change-Id: Ia9c97a63a961eebc336b70d28dc77638144c1834
This commit is contained in:
Lingxian Kong 2020-04-07 10:52:16 +12:00
parent 523d66e8fd
commit aa1d4d2246
242 changed files with 3364 additions and 29701 deletions

View File

@ -14,15 +14,14 @@
- openstack-tox-pylint
- trove-tox-bandit-baseline:
voting: false
- trove-tempest:
- trove-tempest
- trove-functional-mysql:
voting: false
- trove-functional-mysql
- trove-scenario-mysql-single
- trove-scenario-mysql-multi
- trove-scenario-mariadb-single
- trove-scenario-postgresql-single:
- trove-scenario-mysql-single:
voting: false
- trove-scenario-postgresql-multi:
- trove-scenario-mysql-multi:
voting: false
- trove-scenario-mariadb-single:
voting: false
- trove-scenario-mariadb-multi:
voting: false
@ -34,9 +33,12 @@
queue: trove
jobs:
- openstack-tox-pylint
- trove-functional-mysql
- trove-scenario-mysql-single
- trove-scenario-mysql-multi
- trove-functional-mysql:
voting: false
- trove-scenario-mysql-single:
voting: false
- trove-scenario-mysql-multi:
voting: false
experimental:
jobs:
- trove-grenade
@ -145,7 +147,7 @@
trove_resize_time_out: 1800
trove_test_datastore: 'mysql'
trove_test_group: 'mysql'
trove_test_datastore_version: '5.7'
trove_test_datastore_version: '5.7.29'
- job:
name: trove-functional-mysql-nondev
@ -153,11 +155,11 @@
vars:
devstack_localrc:
TROVE_RESIZE_TIME_OUT: 1800
TROVE_NON_DEV_IMAGE_URL_MYSQL: https://tarballs.opendev.org/openstack/trove/images/trove-master-mysql-ubuntu-xenial.qcow2
TROVE_NON_DEV_IMAGE_URL: https://tarballs.opendev.org/openstack/trove/images/trove-master-mysql-ubuntu-xenial.qcow2
trove_resize_time_out: 1800
trove_test_datastore: 'mysql'
trove_test_group: 'mysql'
trove_test_datastore_version: '5.7'
trove_test_datastore_version: '5.7.29'
- job:
name: trove-grenade
@ -212,7 +214,7 @@
vars:
trove_test_datastore: mariadb
trove_test_group: mariadb-supported-single
trove_test_datastore_version: 10.4
trove_test_datastore_version: 10.4.12
devstack_localrc:
TROVE_ENABLE_IMAGE_BUILD: false
@ -222,7 +224,7 @@
vars:
trove_test_datastore: mariadb
trove_test_group: mariadb-supported-multi
trove_test_datastore_version: 10.4
trove_test_datastore_version: 10.4.12
devstack_localrc:
TROVE_ENABLE_IMAGE_BUILD: false
@ -232,7 +234,7 @@
vars:
trove_test_datastore: mysql
trove_test_group: mysql-supported-single
trove_test_datastore_version: 5.7
trove_test_datastore_version: 5.7.29
- job:
name: trove-scenario-mysql-multi
@ -240,7 +242,7 @@
vars:
trove_test_datastore: mysql
trove_test_group: mysql-supported-multi
trove_test_datastore_version: 5.7
trove_test_datastore_version: 5.7.29
- job:
name: trove-scenario-percona-multi

View File

@ -688,10 +688,9 @@ replica_count:
type: integer
replica_of:
description: |
ID or name of an existing instance to replicate
from.
ID or name of an existing instance to replicate from.
in: body
required: false
required: true
type: string
restore_point:
description: |
@ -735,9 +734,10 @@ shard_id:
type: string
slave_of:
description: |
To detach a replica, set ``slave_of`` to null.
To detach a replica, set ``slave_of`` to null. Deprecated in favor of
``replica_of``
in: body
required: true
required: false
type: string
tenant_id:
description: |

View File

@ -1,6 +1,5 @@
{
"instance": {
"replica_of": null,
"slave_of": null
"replica_of": null
}
}

41
backup/Dockerfile Normal file
View File

@ -0,0 +1,41 @@
FROM ubuntu:18.04
LABEL maintainer="anlin.kong@gmail.com"
ARG APTOPTS="-y -qq --no-install-recommends --allow-unauthenticated"
ARG PERCONA_XTRABACKUP_VERSION=24
ENV DEBIAN_FRONTEND noninteractive
ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1
RUN apt-get update \
&& apt-get install $APTOPTS gnupg2 lsb-release apt-utils apt-transport-https ca-certificates software-properties-common curl \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Install percona-xtrabackup for mysql
RUN curl -sSL https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb -o percona-release.deb \
&& dpkg -i percona-release.deb \
&& percona-release enable-only tools release \
&& apt-get update \
&& apt-get install $APTOPTS percona-xtrabackup-${PERCONA_XTRABACKUP_VERSION} \
&& apt-get clean
# Install mariabackup for mariadb
Run apt-key adv --fetch-keys 'https://mariadb.org/mariadb_release_signing_key.asc' \
&& add-apt-repository "deb [arch=amd64] http://mirror2.hs-esslingen.de/mariadb/repo/10.4/ubuntu $(lsb_release -cs) main" \
&& apt-get update \
&& apt-get install $APTOPTS mariadb-backup \
&& apt-get clean
RUN apt-get update \
&& apt-get install $APTOPTS build-essential python3-setuptools python3-all python3-all-dev python3-pip libffi-dev libssl-dev libxml2-dev libxslt1-dev libyaml-dev \
&& apt-get clean
COPY . /opt/trove/backup
WORKDIR /opt/trove/backup
RUN pip3 --no-cache-dir install -U -r requirements.txt
RUN curl -sSL https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 -o /usr/local/bin/dumb-init \
&& chmod +x /usr/local/bin/dumb-init
ENTRYPOINT ["dumb-init", "--single-child", "--"]

207
backup/drivers/base.py Normal file
View File

@ -0,0 +1,207 @@
# Copyright 2020 Catalyst Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import signal
import subprocess
from oslo_config import cfg
from oslo_log import log as logging
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class BaseRunner(object):
"""Base class for Backup Strategy implementations."""
# Subclass should provide the commands.
cmd = None
restore_cmd = None
prepare_cmd = None
encrypt_key = CONF.backup_encryption_key
default_data_dir = '/var/lib/mysql/data'
def __init__(self, *args, **kwargs):
self.process = None
self.pid = None
self.base_filename = kwargs.get('filename')
self.storage = kwargs.pop('storage', None)
self.location = kwargs.pop('location', '')
self.checksum = kwargs.pop('checksum', '')
if 'restore_location' not in kwargs:
kwargs['restore_location'] = self.default_data_dir
self.restore_location = kwargs['restore_location']
self.command = self.cmd % kwargs
self.restore_command = (self.decrypt_cmd +
self.unzip_cmd +
(self.restore_cmd % kwargs))
self.prepare_command = self.prepare_cmd % kwargs
@property
def filename(self):
"""Subclasses may overwrite this to declare a format (.tar)."""
return self.base_filename
@property
def manifest(self):
"""Target file name."""
return "%s%s%s" % (self.filename,
self.zip_manifest,
self.encrypt_manifest)
@property
def zip_cmd(self):
return ' | gzip'
@property
def unzip_cmd(self):
return 'gzip -d -c | '
@property
def zip_manifest(self):
return '.gz'
@property
def encrypt_cmd(self):
return (' | openssl enc -aes-256-cbc -md sha512 -pbkdf2 -iter 10000 '
'-salt -pass pass:%s' %
self.encrypt_key) if self.encrypt_key else ''
@property
def decrypt_cmd(self):
if self.encrypt_key:
return ('openssl enc -d -aes-256-cbc -md sha512 -pbkdf2 -iter '
'10000 -salt -pass pass:%s | '
% self.encrypt_key)
else:
return ''
@property
def encrypt_manifest(self):
return '.enc' if self.encrypt_key else ''
def _run(self):
LOG.info("Running backup cmd: %s", self.command)
self.process = subprocess.Popen(self.command, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid)
self.pid = self.process.pid
def __enter__(self):
"""Start up the process."""
self.pre_backup()
self._run()
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Clean up everything."""
if getattr(self, 'process', None):
try:
# Send a sigterm to the session leader, so that all
# child processes are killed and cleaned up on terminate
os.killpg(self.process.pid, signal.SIGTERM)
self.process.terminate()
except OSError:
pass
if exc_type is not None:
return False
try:
err = self.process.stderr.read()
if err:
raise Exception(err)
except OSError:
pass
if not self.check_process():
raise Exception()
self.post_backup()
return True
def read(self, chunk_size):
return self.process.stdout.read(chunk_size)
def get_metadata(self):
"""Hook for subclasses to get metadata from the backup."""
return {}
def check_process(self):
"""Hook for subclasses to check process for errors."""
return True
def check_restore_process(self):
"""Hook for subclasses to check the restore process for errors."""
return True
def pre_backup(self):
"""Hook for subclasses to run commands before backup."""
pass
def post_backup(self):
"""Hook for subclasses to run commands after backup."""
pass
def pre_restore(self):
"""Hook that is called before the restore command."""
pass
def post_restore(self):
"""Hook that is called after the restore command."""
pass
def unpack(self, location, checksum, command):
stream = self.storage.load(location, checksum)
LOG.info('Running restore from stream, command: %s', command)
self.process = subprocess.Popen(command, shell=True,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
content_length = 0
for chunk in stream:
self.process.stdin.write(chunk)
content_length += len(chunk)
self.process.stdin.close()
try:
err = self.process.stderr.read()
if err:
raise Exception(err)
except OSError:
pass
if not self.check_restore_process():
raise Exception()
return content_length
def run_restore(self):
return self.unpack(self.location, self.checksum, self.restore_command)
def restore(self):
"""Restore backup to data directory.
:returns Restored data size.
"""
self.pre_restore()
content_length = self.run_restore()
self.post_restore()
return content_length

View File

@ -0,0 +1,137 @@
# Copyright 2020 Catalyst Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_log import log as logging
from backup.drivers import mysql_base
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class InnoBackupEx(mysql_base.MySQLBaseRunner):
"""Implementation of Backup and Restore for InnoBackupEx."""
backup_log = '/tmp/innobackupex.log'
prepare_log = '/tmp/prepare.log'
restore_cmd = ('xbstream -x -C %(restore_location)s --parallel=2'
' 2>/tmp/xbstream_extract.log')
prepare_cmd = ('innobackupex'
' --defaults-file=%(restore_location)s/backup-my.cnf'
' --ibbackup=xtrabackup'
' --apply-log'
' %(restore_location)s'
' 2>' + prepare_log)
@property
def cmd(self):
cmd = ('innobackupex'
' --stream=xbstream'
' --parallel=2 ' +
self.user_and_pass + ' %s' % self.default_data_dir +
' 2>' + self.backup_log
)
return cmd + self.zip_cmd + self.encrypt_cmd
def check_restore_process(self):
"""Check whether xbstream restore is successful."""
LOG.info('Checking return code of xbstream restore process.')
return_code = self.process.wait()
if return_code != 0:
LOG.error('xbstream exited with %s', return_code)
return False
with open('/tmp/xbstream_extract.log', 'r') as xbstream_log:
for line in xbstream_log:
# Ignore empty lines
if not line.strip():
continue
LOG.error('xbstream restore failed with: %s',
line.rstrip('\n'))
return False
return True
def post_restore(self):
"""Hook that is called after the restore command."""
LOG.info("Running innobackupex prepare: %s.", self.prepare_command)
processutils.execute(self.prepare_command, shell=True)
LOG.info("Checking innobackupex prepare log")
with open(self.prepare_log, 'r') as prepare_log:
output = prepare_log.read()
if not output:
msg = "innobackupex prepare log file empty"
raise Exception(msg)
last_line = output.splitlines()[-1].strip()
if not re.search('completed OK!', last_line):
msg = "innobackupex prepare did not complete successfully"
raise Exception(msg)
class InnoBackupExIncremental(InnoBackupEx):
"""InnoBackupEx incremental backup."""
incremental_prep = ('innobackupex'
' --defaults-file=%(restore_location)s/backup-my.cnf'
' --ibbackup=xtrabackup'
' --apply-log'
' --redo-only'
' %(restore_location)s'
' %(incremental_args)s'
' 2>/tmp/innoprepare.log')
def __init__(self, *args, **kwargs):
if not kwargs.get('lsn'):
raise AttributeError('lsn attribute missing')
self.parent_location = kwargs.pop('parent_location', '')
self.parent_checksum = kwargs.pop('parent_checksum', '')
self.restore_content_length = 0
super(InnoBackupExIncremental, self).__init__(*args, **kwargs)
@property
def cmd(self):
cmd = ('innobackupex'
' --stream=xbstream'
' --incremental'
' --incremental-lsn=%(lsn)s ' +
self.user_and_pass + ' %s' % self.default_data_dir +
' 2>' + self.backup_log)
return cmd + self.zip_cmd + self.encrypt_cmd
def get_metadata(self):
_meta = super(InnoBackupExIncremental, self).get_metadata()
_meta.update({
'parent_location': self.parent_location,
'parent_checksum': self.parent_checksum,
})
return _meta
def run_restore(self):
"""Run incremental restore.
First grab all parents and prepare them with '--redo-only'. After
all backups are restored the super class InnoBackupEx post_restore
method is called to do the final prepare with '--apply-log'
"""
LOG.debug('Running incremental restore')
self.incremental_restore(self.location, self.checksum)
return self.restore_content_length

View File

@ -0,0 +1,87 @@
# Copyright 2020 Catalyst Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from oslo_log import log as logging
from backup.drivers import mysql_base
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class MariaBackup(mysql_base.MySQLBaseRunner):
"""Implementation of Backup and Restore using mariabackup."""
backup_log = '/tmp/mariabackup.log'
restore_log = '/tmp/mbstream_extract.log'
restore_cmd = ('mbstream -x -C %(restore_location)s 2>' + restore_log)
prepare_cmd = ''
@property
def cmd(self):
cmd = ('mariabackup --backup --stream=xbstream ' +
self.user_and_pass + ' 2>' + self.backup_log)
return cmd + self.zip_cmd + self.encrypt_cmd
def check_restore_process(self):
LOG.debug('Checking return code of mbstream restore process.')
return_code = self.process.wait()
if return_code != 0:
LOG.error('mbstream exited with %s', return_code)
return False
return True
class MariaBackupIncremental(MariaBackup):
"""Incremental backup and restore using mariabackup."""
incremental_prep = ('mariabackup --prepare '
'--target-dir=%(restore_location)s '
'%(incremental_args)s '
'2>/tmp/innoprepare.log')
def __init__(self, *args, **kwargs):
if not kwargs.get('lsn'):
raise AttributeError('lsn attribute missing')
self.parent_location = kwargs.pop('parent_location', '')
self.parent_checksum = kwargs.pop('parent_checksum', '')
self.restore_content_length = 0
super(MariaBackupIncremental, self).__init__(*args, **kwargs)
@property
def cmd(self):
cmd = (
'mariabackup --backup --stream=xbstream'
' --incremental-lsn=%(lsn)s ' +
self.user_and_pass +
' 2>' +
self.backup_log
)
return cmd + self.zip_cmd + self.encrypt_cmd
def get_metadata(self):
meta = super(MariaBackupIncremental, self).get_metadata()
meta.update({
'parent_location': self.parent_location,
'parent_checksum': self.parent_checksum,
})
return meta
def run_restore(self):
"""Run incremental restore."""
LOG.debug('Running incremental restore')
self.incremental_restore(self.location, self.checksum)
return self.restore_content_length

View File

@ -0,0 +1,139 @@
# Copyright 2020 Catalyst Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import re
import shutil
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_log import log as logging
from backup.drivers import base
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class MySQLBaseRunner(base.BaseRunner):
def __init__(self, *args, **kwargs):
super(MySQLBaseRunner, self).__init__(*args, **kwargs)
@property
def user_and_pass(self):
return ('--user=%(user)s --password=%(password)s --host=%(host)s' %
{'user': CONF.db_user,
'password': CONF.db_password,
'host': CONF.db_host})
@property
def filename(self):
return '%s.xbstream' % self.base_filename
def check_process(self):
"""Check the backup output for 'completed OK!'."""
LOG.debug('Checking backup process output.')
with open(self.backup_log, 'r') as backup_log:
output = backup_log.read()
if not output:
LOG.error("Backup log file %s empty.", self.backup_log)
return False
last_line = output.splitlines()[-1].strip()
if not re.search('completed OK!', last_line):
LOG.error("Backup did not complete successfully.")
return False
return True
def get_metadata(self):
LOG.debug('Getting metadata for backup %s', self.base_filename)
meta = {}
lsn = re.compile(r"The latest check point \(for incremental\): "
r"'(\d+)'")
with open(self.backup_log, 'r') as backup_log:
output = backup_log.read()
match = lsn.search(output)
if match:
meta = {'lsn': match.group(1)}
LOG.info("Updated metadata for backup %s: %s", self.base_filename,
meta)
return meta
def incremental_restore_cmd(self, incremental_dir):
"""Return a command for a restore with a incremental location."""
args = {'restore_location': incremental_dir}
return (self.decrypt_cmd + self.unzip_cmd + self.restore_cmd % args)
def incremental_prepare_cmd(self, incremental_dir):
if incremental_dir is not None:
incremental_arg = '--incremental-dir=%s' % incremental_dir
else:
incremental_arg = ''
args = {
'restore_location': self.restore_location,
'incremental_args': incremental_arg,
}
return self.incremental_prep % args
def incremental_prepare(self, incremental_dir):
prepare_cmd = self.incremental_prepare_cmd(incremental_dir)
LOG.info("Running restore prepare command: %s.", prepare_cmd)
processutils.execute(prepare_cmd, shell=True)
def incremental_restore(self, location, checksum):
"""Recursively apply backups from all parents.
If we are the parent then we restore to the restore_location and
we apply the logs to the restore_location only.
Otherwise if we are an incremental we restore to a subfolder to
prevent stomping on the full restore data. Then we run apply log
with the '--incremental-dir' flag
:param location: The source backup location.
:param checksum: Checksum of the source backup for validation.
"""
metadata = self.storage.load_metadata(location, checksum)
incremental_dir = None
if 'parent_location' in metadata:
LOG.info("Restoring parent: %(parent_location)s"
" checksum: %(parent_checksum)s.", metadata)
parent_location = metadata['parent_location']
parent_checksum = metadata['parent_checksum']
# Restore parents recursively so backup are applied sequentially
self.incremental_restore(parent_location, parent_checksum)
# for *this* backup set the incremental_dir
# just use the checksum for the incremental path as it is
# sufficiently unique /var/lib/mysql/<checksum>
incremental_dir = os.path.join('/var/lib/mysql', checksum)
os.makedirs(incremental_dir)
command = self.incremental_restore_cmd(incremental_dir)
else:
# The parent (full backup) use the same command from InnobackupEx
# super class and do not set an incremental_dir.
command = self.restore_command
self.restore_content_length += self.unpack(location, checksum, command)
self.incremental_prepare(incremental_dir)
# Delete after restoring this part of backup
if incremental_dir:
shutil.rmtree(incremental_dir)

149
backup/main.py Normal file
View File

@ -0,0 +1,149 @@
# Copyright 2020 Catalyst Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import importutils
import sys
topdir = os.path.normpath(
os.path.join(os.path.abspath(sys.argv[0]), os.pardir, os.pardir))
sys.path.insert(0, topdir)
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
cli_opts = [
cfg.StrOpt('backup-id'),
cfg.StrOpt(
'storage-driver',
default='swift',
choices=['swift']
),
cfg.StrOpt(
'driver',
default='innobackupex',
choices=['innobackupex', 'xtrabackup', 'mariabackup']
),
cfg.BoolOpt('backup'),
cfg.StrOpt('backup-encryption-key'),
cfg.StrOpt('db-user'),
cfg.StrOpt('db-password'),
cfg.StrOpt('db-host'),
cfg.StrOpt('os-token'),
cfg.StrOpt('os-auth-url'),
cfg.StrOpt('os-tenant-id'),
cfg.StrOpt('swift-container', default='database_backups'),
cfg.DictOpt('swift-extra-metadata'),
cfg.StrOpt('restore-from'),
cfg.StrOpt('restore-checksum'),
cfg.BoolOpt('incremental'),
cfg.StrOpt('parent-location'),
cfg.StrOpt(
'parent-checksum',
help='It is up to the storage driver to decide to validate the '
'checksum or not. '
),
]
driver_mapping = {
'innobackupex': 'backup.drivers.innobackupex.InnoBackupEx',
'innobackupex_inc': 'backup.drivers.innobackupex.InnoBackupExIncremental',
'mariabackup': 'backup.drivers.mariabackup.MariaBackup',
'mariabackup_inc': 'backup.drivers.mariabackup.MariaBackupIncremental',
}
storage_mapping = {
'swift': 'backup.storage.swift.SwiftStorage',
}
def stream_backup_to_storage(runner_cls, storage):
parent_metadata = {}
if CONF.incremental:
if not CONF.parent_location:
LOG.error('--parent-location should be provided for incremental '
'backup')
exit(1)
parent_metadata = storage.load_metadata(CONF.parent_location,
CONF.parent_checksum)
parent_metadata.update(
{
'parent_location': CONF.parent_location,
'parent_checksum': CONF.parent_checksum
}
)
try:
with runner_cls(filename=CONF.backup_id, **parent_metadata) as bkup:
checksum, location = storage.save(
bkup,
metadata=CONF.swift_extra_metadata
)
LOG.info('Backup successfully, checksum: %s, location: %s',
checksum, location)
except Exception as err:
LOG.exception('Failed to call stream_backup_to_storage, error: %s',
err)
def stream_restore_from_storage(runner_cls, storage):
lsn = ""
if storage.is_incremental_backup(CONF.restore_from):
lsn = storage.get_backup_lsn(CONF.restore_from)
try:
runner = runner_cls(storage=storage, location=CONF.restore_from,
checksum=CONF.restore_checksum, lsn=lsn)
restore_size = runner.restore()
LOG.info('Restore successfully, restore_size: %s', restore_size)
except Exception as err:
LOG.exception('Failed to call stream_restore_from_storage, error: %s',
err)
def main():
CONF.register_cli_opts(cli_opts)
logging.register_options(CONF)
CONF(sys.argv[1:], project='trove-backup')
logging.setup(CONF, 'trove-backup')
runner_cls = importutils.import_class(driver_mapping[CONF.driver])
storage = importutils.import_class(storage_mapping[CONF.storage_driver])()
if CONF.backup:
if CONF.incremental:
runner_cls = importutils.import_class(
driver_mapping['%s_inc' % CONF.driver])
LOG.info('Starting backup database to %s, backup ID %s',
CONF.storage_driver, CONF.backup_id)
stream_backup_to_storage(runner_cls, storage)
else:
if storage.is_incremental_backup(CONF.restore_from):
LOG.debug('Restore from incremental backup')
runner_cls = importutils.import_class(
driver_mapping['%s_inc' % CONF.driver])
LOG.info('Starting restore database from %s, location: %s',
CONF.storage_driver, CONF.restore_from)
stream_restore_from_storage(runner_cls, storage)
if __name__ == '__main__':
sys.exit(main())

6
backup/requirements.txt Normal file
View File

@ -0,0 +1,6 @@
oslo.config!=4.3.0,!=4.4.0;python_version>='3.0' # Apache-2.0
oslo.log;python_version>='3.0' # Apache-2.0
oslo.utils!=3.39.1,!=3.40.0,!=3.40.1;python_version>='3.0' # Apache-2.0
oslo.concurrency;python_version>='3.0' # Apache-2.0
keystoneauth1 # Apache-2.0
python-swiftclient # Apache-2.0

48
backup/storage/base.py Normal file
View File

@ -0,0 +1,48 @@
# Copyright 2020 Catalyst Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
class Storage(object):
"""Base class for Storage driver implementation."""
@abc.abstractmethod
def save(self, stream, metadata=None, **kwargs):
"""Persist information from the stream.
Should return the new backup checkshum and location.
"""
@abc.abstractmethod
def load(self, location, backup_checksum, **kwargs):
"""Load a stream from the data location.
Should return an object that provides "read" method.
"""
def load_metadata(self, parent_location, parent_checksum):
"""Load metadata for a parent backup.
It's up to the storage driver to decide how to implement this function.
"""
return {}
def is_incremental_backup(self, location):
"""Check if the location is an incremental backup."""
return False
@abc.abstractmethod
def get_backup_lsn(self, location):
"""Get the backup LSN."""

294
backup/storage/swift.py Normal file
View File

@ -0,0 +1,294 @@
# Copyright 2020 Catalyst Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
import json
from keystoneauth1 import session
from keystoneauth1.identity import v3
from oslo_config import cfg
from oslo_log import log as logging
import swiftclient
from backup.storage import base
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
def _get_user_keystone_session(auth_url, token, tenant_id):
auth = v3.Token(
auth_url=auth_url, token=token,
project_domain_name="Default",
project_id=tenant_id
)
return session.Session(auth=auth, verify=False)
def _get_service_client(auth_url, token, tenant_id):
sess = _get_user_keystone_session(auth_url, token, tenant_id)
return swiftclient.Connection(session=sess)
def _set_attr(original):
"""Return a swift friendly header key."""
key = original.replace('_', '-')
return 'X-Object-Meta-%s' % key
def _get_attr(original):
"""Get a friendly name from an object header key."""
key = original.replace('-', '_')
key = key.replace('x_object_meta_', '')
return key
class StreamReader(object):
"""Wrap the stream from the backup process and chunk it into segements."""
def __init__(self, stream, container, filename, max_file_size):
self.stream = stream
self.container = container
self.filename = filename
self.max_file_size = max_file_size
self.segment_length = 0
self.process = None
self.file_number = 0
self.end_of_file = False
self.end_of_segment = False
self.segment_checksum = hashlib.md5()
@property
def base_filename(self):
"""Filename with extensions removed."""
return self.filename.split('.')[0]
@property
def segment(self):
return '%s_%08d' % (self.base_filename, self.file_number)
@property
def first_segment(self):
return '%s_%08d' % (self.base_filename, 0)
@property
def segment_path(self):
return '%s/%s' % (self.container, self.segment)
def read(self, chunk_size=2 ** 16):
if self.end_of_segment:
self.segment_length = 0
self.segment_checksum = hashlib.md5()
self.end_of_segment = False
# Upload to a new file if we are starting or too large
if self.segment_length > (self.max_file_size - chunk_size):
self.file_number += 1
self.end_of_segment = True
return ''
chunk = self.stream.read(chunk_size)
if not chunk:
self.end_of_file = True
return ''
self.segment_checksum.update(chunk)
self.segment_length += len(chunk)
return chunk
class SwiftStorage(base.Storage):
def __init__(self):
self.client = _get_service_client(CONF.os_auth_url, CONF.os_token,
CONF.os_tenant_id)
def save(self, stream, metadata=None, container='database_backups'):
"""Persist data from the stream to swift.
* Read data from stream, upload to swift
* Update the new object metadata, stream provides method to get
metadata.
:returns the new object checkshum and swift full URL.
"""
filename = stream.manifest
LOG.info('Saving %(filename)s to %(container)s in swift.',
{'filename': filename, 'container': container})
# Create the container if it doesn't already exist
LOG.debug('Ensuring container %s', container)
self.client.put_container(container)
# Swift Checksum is the checksum of the concatenated segment checksums
swift_checksum = hashlib.md5()
# Wrap the output of the backup process to segment it for swift
stream_reader = StreamReader(stream, container, filename,
2 * (1024 ** 3))
url = self.client.url
# Full location where the backup manifest is stored
location = "%s/%s/%s" % (url, container, filename)
LOG.info('Uploading to %s', location)
# Information about each segment upload job
segment_results = []
# Read from the stream and write to the container in swift
while not stream_reader.end_of_file:
LOG.debug('Uploading segment %s.', stream_reader.segment)
path = stream_reader.segment_path
etag = self.client.put_object(container,
stream_reader.segment,
stream_reader)
segment_checksum = stream_reader.segment_checksum.hexdigest()
# Check each segment MD5 hash against swift etag
if etag != segment_checksum:
msg = ('Failed to upload data segment to swift. ETAG: %(tag)s '
'Segment MD5: %(checksum)s.' %
{'tag': etag, 'checksum': segment_checksum})
raise Exception(msg)
segment_results.append({
'path': path,
'etag': etag,
'size_bytes': stream_reader.segment_length
})
swift_checksum.update(segment_checksum.encode())
# All segments uploaded.
num_segments = len(segment_results)
LOG.debug('File uploaded in %s segments.', num_segments)
# An SLO will be generated if the backup was more than one segment in
# length.
large_object = num_segments > 1
# Meta data is stored as headers
if metadata is None:
metadata = {}
metadata.update(stream.get_metadata())
headers = {}
for key, value in metadata.items():
headers[_set_attr(key)] = value
LOG.debug('Metadata headers: %s', headers)
if large_object:
manifest_data = json.dumps(segment_results)
LOG.info('Creating the SLO manifest file, manifest content: %s',
manifest_data)
# The etag returned from the manifest PUT is the checksum of the
# manifest object (which is empty); this is not the checksum we
# want.
self.client.put_object(container,
filename,
manifest_data,
query_string='multipart-manifest=put')
# Validation checksum is the Swift Checksum
final_swift_checksum = swift_checksum.hexdigest()
else:
LOG.info('Moving segment %(segment)s to %(filename)s.',
{'segment': stream_reader.first_segment,
'filename': filename})
segment_result = segment_results[0]
# Just rename it via a special put copy.
headers['X-Copy-From'] = segment_result['path']
self.client.put_object(container,
filename, '',
headers=headers)
# Delete the old segment file that was copied
LOG.debug('Deleting the old segment file %s.',
stream_reader.first_segment)
self.client.delete_object(container,
stream_reader.first_segment)
final_swift_checksum = segment_result['etag']
# Validate the object by comparing checksums
resp = self.client.head_object(container, filename)
# swift returns etag in double quotes
# e.g. '"dc3b0827f276d8d78312992cc60c2c3f"'
etag = resp['etag'].strip('"')
# Raise an error and mark backup as failed
if etag != final_swift_checksum:
msg = ('Failed to upload data to swift. Manifest ETAG: %(tag)s '
'Swift MD5: %(checksum)s' %
{'tag': etag, 'checksum': final_swift_checksum})
raise Exception(msg)
return (final_swift_checksum, location)
def _explodeLocation(self, location):
storage_url = "/".join(location.split('/')[:-2])
container = location.split('/')[-2]
filename = location.split('/')[-1]
return storage_url, container, filename
def _verify_checksum(self, etag, checksum):
etag_checksum = etag.strip('"')
if etag_checksum != checksum:
msg = ('Checksum validation failure, actual: %s, expected: %s' %
(etag_checksum, checksum))
raise Exception(msg)
def load(self, location, backup_checksum):
"""Get object from the location."""
storage_url, container, filename = self._explodeLocation(location)
headers, contents = self.client.get_object(container, filename,
resp_chunk_size=2 ** 16)
if backup_checksum:
self._verify_checksum(headers.get('etag', ''), backup_checksum)
return contents
def load_metadata(self, parent_location, parent_checksum):
"""Load metadata from swift."""
if not parent_location:
return {}
_, container, filename = self._explodeLocation(parent_location)
headers = self.client.head_object(container, filename)
if parent_checksum:
self._verify_checksum(headers.get('etag', ''), parent_checksum)
_meta = {}
for key, value in headers.items():
if key.startswith('x-object-meta'):
_meta[_get_attr(key)] = value
return _meta
def is_incremental_backup(self, location):
"""Check if the location is an incremental backup."""
_, container, filename = self._explodeLocation(location)
headers = self.client.head_object(container, filename)
if 'x-object-meta-parent-location' in headers:
return True
return False
def get_backup_lsn(self, location):
"""Get the backup LSN."""
_, container, filename = self._explodeLocation(location)
headers = self.client.head_object(container, filename)
return headers.get('x-object-meta-lsn')

View File

@ -362,7 +362,7 @@ function create_subnet_v6 {
if [[ -n "$IPV6_PRIVATE_NETWORK_GATEWAY" ]]; then
subnet_params+="--gateway $IPV6_PRIVATE_NETWORK_GATEWAY "
fi
if [ -n $SUBNETPOOL_V6_ID ]; then
if [[ -n $SUBNETPOOL_V6_ID ]]; then
subnet_params+="--subnet-pool $SUBNETPOOL_V6_ID "
else
subnet_params+="--subnet-range $FIXED_RANGE_V6 $ipv6_modes} "
@ -447,27 +447,26 @@ function create_guest_image {
return 0
fi
image_name="trove-datastore-${TROVE_IMAGE_OS}-${TROVE_IMAGE_OS_RELEASE}-${TROVE_DATASTORE_TYPE}"
image_url_var="TROVE_NON_DEV_IMAGE_URL_${TROVE_DATASTORE_TYPE^^}"
image_url=`eval echo '$'"$image_url_var"`
image_name="trove-guest-${TROVE_IMAGE_OS}-${TROVE_IMAGE_OS_RELEASE}"
mkdir -p $HOME/images
image_file=$HOME/images/${image_name}.qcow2
if [[ -n ${image_url} ]]; then
echo "Downloading guest image from ${image_url}"
curl -sSL ${image_url} -o ${image_file}
if [[ -n ${TROVE_NON_DEV_IMAGE_URL} ]]; then
echo "Downloading guest image from ${TROVE_NON_DEV_IMAGE_URL}"
curl -sSL ${TROVE_NON_DEV_IMAGE_URL} -o ${image_file}
else
echo "Starting to create guest image"
TROVE_BRANCH=${TROVE_BRANCH} $DEST/trove/integration/scripts/trovestack \
$DEST/trove/integration/scripts/trovestack \
build-image \
${TROVE_DATASTORE_TYPE} \
${TROVE_IMAGE_OS} \
${TROVE_IMAGE_OS_RELEASE} \
true
true \
${TROVE_IMAGE_OS} \
${image_file}
fi
if [ ! -f ${image_file} ]; then
if [[ ! -f ${image_file} ]]; then
echo "Image file was not found at ${image_file}"
exit 1
fi
@ -485,7 +484,7 @@ function create_guest_image {
$TROVE_MANAGE datastore_update $TROVE_DATASTORE_TYPE $TROVE_DATASTORE_VERSION
echo "Add parameter validation rules if available"
if [ -f $DEST/trove/trove/templates/$TROVE_DATASTORE_TYPE/validation-rules.json ]; then
if [[ -f $DEST/trove/trove/templates/$TROVE_DATASTORE_TYPE/validation-rules.json ]]; then
$TROVE_MANAGE db_load_datastore_config_parameters "$TROVE_DATASTORE_TYPE" "$TROVE_DATASTORE_VERSION" \
$DEST/trove/trove/templates/$TROVE_DATASTORE_TYPE/validation-rules.json
fi

View File

@ -29,15 +29,9 @@ TROVE_LOCAL_API_PASTE_INI=${TROVE_LOCAL_API_PASTE_INI:-${TROVE_LOCAL_CONF_DIR}/a
TROVE_LOCAL_POLICY_JSON=${TROVE_LOCAL_POLICY_JSON:-${TROVE_LOCAL_CONF_DIR}/policy.json}
TROVE_IMAGE_OS=${TROVE_IMAGE_OS:-"ubuntu"}
TROVE_IMAGE_OS_RELEASE=${TROVE_IMAGE_OS_RELEASE:-"xenial"}
TROVE_IMAGE_OS_RELEASE=${TROVE_IMAGE_OS_RELEASE:-"bionic"}
TROVE_DATASTORE_TYPE=${TROVE_DATASTORE_TYPE:-"mysql"}
if [[ "$DISTRO" == "xenial" || "$DISTRO" == "bionic" ]]; then
TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"5.7"}
TROVE_DATASTORE_PACKAGE=${TROVE_DATASTORE_PACKAGE:-"mysql-server-5.7"}
else
TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"5.6"}
TROVE_DATASTORE_PACKAGE=${TROVE_DATASTORE_PACKAGE:-"mysql-server-5.6"}
fi
TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"5.7.29"}
# Configuration values listed here for reference
TROVE_MAX_ACCEPTED_VOLUME_SIZE=${TROVE_MAX_ACCEPTED_VOLUME_SIZE}
@ -46,8 +40,8 @@ TROVE_MAX_VOLUMES_PER_TENANT=${TROVE_MAX_VOLUMES_PER_TENANT}
TROVE_AGENT_CALL_LOW_TIMEOUT=${TROVE_AGENT_CALL_LOW_TIMEOUT}
TROVE_AGENT_CALL_HIGH_TIMEOUT=${TROVE_AGENT_CALL_HIGH_TIMEOUT:-1200}
TROVE_RESIZE_TIME_OUT=${TROVE_RESIZE_TIME_OUT}
TROVE_USAGE_TIMEOUT=${TROVE_USAGE_TIMEOUT:-900}
TROVE_STATE_CHANGE_WAIT_TIME=${TROVE_STATE_CHANGE_WAIT_TIME}
TROVE_USAGE_TIMEOUT=${TROVE_USAGE_TIMEOUT:-1800}
TROVE_STATE_CHANGE_WAIT_TIME=${TROVE_STATE_CHANGE_WAIT_TIME:-600}
TROVE_COMMAND_PROCESS_TIMEOUT=${TROVE_COMMAND_PROCESS_TIMEOUT:-60}
# Set up the host gateway
@ -90,4 +84,4 @@ CELLSV2_SETUP=singleconductor
# Enable or disable the Trove guest image build during devstack installation.
TROVE_ENABLE_IMAGE_BUILD=${TROVE_ENABLE_IMAGE_BUILD:-"true"}
TROVE_NON_DEV_IMAGE_URL_MYSQL=${TROVE_NON_DEV_IMAGE_URL_MYSQL:-""}
TROVE_NON_DEV_IMAGE_URL=${TROVE_NON_DEV_IMAGE_URL:-""}

View File

@ -26,54 +26,42 @@ stored in Glance. This document shows you the steps to build the guest images.
periodically built and published in
http://tarballs.openstack.org/trove/images/ in Trove upstream CI.
Additionally, if you install Trove in devstack environment, a MySQL image
Additionally, if you install Trove in devstack environment, the guest image
is created and registered in Glance automatically, unless it's disabled by
setting ``TROVE_ENABLE_IMAGE_BUILD=false`` in devstack local.conf file.
High Level Overview of a Trove Guest Instance
=============================================
At the most basic level, a Trove Guest Instance is a Nova instance
launched by Trove in response to a create command. For most of this
document, we will confine ourselves to single instance databases; in
other words, without the additional complexity of replication or
mirroring. Guest instances and Guest images for replicated and
mirrored database instances will be addressed specifically in later
sections of this document.
At the most basic level, a Trove Guest Instance is a Nova instance launched by
Trove in response to a create command. This section describes the various
components of a Trove Guest Instance.
This section describes the various components of a Trove Guest
Instance.
----------------
Operating System
----------------
-----------------------------
Operating System and Database
-----------------------------
The officially supported operating system is Ubuntu, based on which the
functional tests are running.
A Trove Guest Instance contains at least a functioning Operating
System and the database software that the instance wishes to provide
(as a Service). For example, if your chosen operating system is Ubuntu
and you wish to deliver MySQL version 5.7, then your guest instance is
a Nova instance running the Ubuntu operating system and will have
MySQL version 5.7 installed on it.
------
Docker
------
Since Vitoria release, all the datastore services are installed by docker
container inside the Trove instance, so docker should be installed when
building the guest image.
-----------------
Trove Guest Agent
-----------------
Trove supports multiple databases, some of them are relational (RDBMS)
and some are non-relational (NoSQL). In order to provide a common
management interface to all of these, the Trove Guest Instance has on
it a 'Guest Agent'. The Trove Guest Agent is a component of the
Trove system that is specific to the database running on that Guest
Instance.
The guest agent runs inside the Nova instances that are used to run the
database engines. The agent listens to the messaging bus for the topic and is
responsible for actually translating and executing the commands that are sent
to it by the task manager component for the particular datastore.
The purpose of the Trove Guest Agent is to implement the Trove Guest
Agent API for the specific database. This includes such things as the
implementation of the database 'start' and 'stop' commands. The Trove
Guest Agent API is the common API used by Trove to communicate with
any guest database, and the Guest Agent is the implementation of that
API for the specific database.
The Trove Guest Agent runs inside the Trove Guest Instance.
Trove guest agent is responsible for datastore docker container management.
------------------------------------------
Injected Configuration for the Guest Agent
@ -104,44 +92,45 @@ services(e.g. the message queue).
Building Guest Images
=====================
Since Victoria release, a single trove guest image can be used for different
datastores, it's unnecessary to maintain different images for differnt
datastores.
-----------------------------
Build images using trovestack
-----------------------------
``trovestack`` is the recommended tooling provided by Trove community to build
the guest images. Before running ``trovestack`` command, go to the scripts
folder:
the guest images. Before running ``trovestack`` command:
.. code-block:: console
git clone https://opendev.org/openstack/trove
cd trove/integration/scripts
The trove guest agent image could be created by running the following command:
The trove guest image could be created by running the following command:
.. code-block:: console
$ ./trovestack build-image \
${datastore_type} \
${guest_os} \
${guest_os_release} \
${dev_mode} \
${guest_username} \
${imagepath}
${output_image_path}
* Currently, only ``guest_os=ubuntu`` and ``guest_os_release=xenial`` are fully
* Currently, only ``guest_os=ubuntu`` and ``guest_os_release=bionic`` are fully
tested and supported.
* Default input values:
.. code-block:: ini
datastore_type=mysql
guest_os=ubuntu
guest_os_release=xenial
guest_os_release=bionic
dev_mode=true
guest_username=ubuntu
imagepath=$HOME/images/trove-${guest_os}-${guest_os_release}-${datastore_type}
output_image_path=$HOME/images/trove-guest--${guest_os}-${guest_os_release}-dev
* ``dev_mode=true`` is mainly for testing purpose for trove developers and it's
necessary to build the image on the trove controller host, because the host
@ -159,31 +148,27 @@ The trove guest agent image could be created by running the following command:
* ``HOST_SCP_USERNAME``: Only used in dev mode, this is the user name used by
guest agent to connect to the controller host, e.g. in devstack
environment, it should be the ``stack`` user.
* ``GUEST_WORKING_DIR``: The place to save the guest image, default value is
``$HOME/images``.
* ``TROVE_BRANCH``: Only used in dev mode. The branch name of Trove code
repository, by default it's master, use other branches as needed such as
stable/train.
For example, in order to build a MySQL image for Ubuntu Xenial operating
For example, in order to build a guest image for Ubuntu Bionic operating
system in development mode:
.. code-block:: console
$ ./trovestack build-image mysql ubuntu xenial true
$ ./trovestack build-image ubuntu bionic true ubuntu
Once the image build is finished, the cloud administrator needs to register the
image in Glance and register a new datastore or version in Trove using
``trove-manage`` command, e.g. after building an image for MySQL 5.7.1:
``trove-manage`` command, e.g. after building an image for MySQL 5.7.29:
.. code-block:: console
$ openstack image create ubuntu-mysql-5.7.1-dev \
--public \
$ openstack image create trove-guest-ubuntu-bionic \
--private \
--disk-format qcow2 \
--container-format bare \
--file ~/images/ubuntu-xenial-mysql.qcow2
$ trove-manage datastore_version_update mysql 5.7.1 mysql $image_id "" 1
--file ~/images/trove-guest-ubuntu-bionic-dev.qcow2
$ trove-manage datastore_version_update mysql 5.7.29 mysql $image_id "" 1
$ trove-manage db_load_datastore_config_parameters mysql 5.7.29 ${trove_repo_dir}/trove/templates/mysql/validation-rules.json
If you see anything error or need help for the image creation, please ask help
either in ``#openstack-trove`` IRC channel or sending emails to

View File

@ -322,7 +322,7 @@ Command examples:
# Use 5.7.29 as the default datastore version for 'mysql'
trove-manage datastore_update mysql 5.7.29
# Register configuration parameters for 5.7.29 version of datastore 'mysql'
trove-manage db_load_datastore_config_parameters mysql 5.7.29 $workdir/trove/trove/templates/mysql/validation-rules.json
trove-manage db_load_datastore_config_parameters mysql 5.7.29 ${trove_repo_dir}}/trove/templates/mysql/validation-rules.json
Quota Management

View File

@ -1,3 +0,0 @@
These cloudinit scripts will used as userdata on instance create
File names should match pattern: service_type.cloudinit
For example: mysql.cloudinit

View File

@ -1,4 +0,0 @@
These conf files are read and used by the guest to provide extra
information to the guest. The first example of this is the
guest_info.conf which will have the uuid of the instance so that
the guest can report back things to the infra.

View File

@ -1 +0,0 @@
# Guest-specific information injected by the taskmanager

View File

@ -1,166 +0,0 @@
[DEFAULT]
#=========== RPC Configuration ======================
# URL representing the messaging driver to use and its full configuration.
# If not set, we fall back to the 'rpc_backend' option and driver specific
# configuration.
#transport_url=<None>
# The messaging driver to use. Options include rabbit, qpid and zmq.
# Default is rabbit. (string value)
#rpc_backend=rabbit
# The default exchange under which topics are scoped. May be
# overridden by an exchange name specified in the 'transport_url option.
control_exchange = trove
# ========== Configuration options for Swift ==========
# The swift_url can be specified directly or fetched from Keystone catalog.
# To fetch from Keystone, comment out swift_url, and uncomment the others.
# swift_url = http://10.0.0.1:8080/v1/AUTH_
# Region name of this node. Default value is None.
# os_region_name = RegionOne
# Service type to use when searching catalog.
# swift_service_type = object-store
# ========== Datastore Manager Configurations ==========
# Datastore manager implementations.
# Format: list of 'datastore-type:datastore.manager.implementation.module'
# datastore_registry_ext = mysql:trove.guestagent.datastore.mysql.manager.Manager, percona:trove.guestagent.datastore.mysql.manager.Manager
# ========== Default Users / DBs Configuration ==========
# Permissions to grant "root" user by default
root_grant = ALL
root_grant_option = True
# root_grant = ALTER ROUTINE, CREATE, ALTER, CREATE ROUTINE, CREATE TEMPORARY TABLES, CREATE VIEW, CREATE USER, DELETE, DROP, EVENT, EXECUTE, INDEX, INSERT, LOCK TABLES, PROCESS, REFERENCES, SELECT, SHOW DATABASES, SHOW VIEW, TRIGGER, UPDATE, USAGE
# root_grant_option = False
# Default password Length for root password
# default_password_length = 36
# ========== Default Storage Options for backup ==========
# Default configuration for storage strategy and storage options
# for backups
# For storage to Swift, use the following as defaults:
# storage_strategy = SwiftStorage
# storage_namespace = trove.common.strategies.storage.swift
# Default config options for storing backups to swift
# backup_swift_container = database_backups
# backup_use_gzip_compression = True
# backup_use_openssl_encryption = True
# backup_aes_cbc_key = "default_aes_cbc_key"
# backup_use_snet = False
# backup_chunk_size = 65536
# backup_segment_max_size = 2147483648
# ========== Sample Logging Configuration ==========
# Show debugging output in logs (sets DEBUG log level output)
# debug = True
# Directory and path for log files
log_dir = /var/log/trove/
log_file = logfile.txt
log_config_append = /etc/trove/trove-logging-guestagent.conf
[profiler]
# If False fully disable profiling feature.
#enabled = False
# If False doesn't trace SQL requests.
#trace_sqlalchemy = True
[oslo_messaging_notifications]
#
# From oslo.messaging
#
# The Driver(s) to handle sending notifications. Possible
# values are messaging, messagingv2, routing, log, test, noop
# (multi valued)
# Deprecated group/name - [DEFAULT]/notification_driver
#driver =
# A URL representing the messaging driver to use for
# notifications. If not set, we fall back to the same
# configuration used for RPC. (string value)
# Deprecated group/name - [DEFAULT]/notification_transport_url
#transport_url = <None>
# AMQP topic used for OpenStack notifications. (list value)
# Deprecated group/name - [rpc_notifier2]/topics
# Deprecated group/name - [DEFAULT]/notification_topics
#topics = notifications
# The maximum number of attempts to re-send a notification
# message which failed to be delivered due to a recoverable
# error. 0 - No retry, -1 - indefinite (integer value)
#retry = -1
# ========== Datastore Specific Configuration Options ==========
[mysql]
# For mysql, the following are the defaults for backup, and restore:
# backup_strategy = InnoBackupEx
# backup_namespace = trove.guestagent.strategies.backup.mysql_impl
# restore_namespace = trove.guestagent.strategies.restore.mysql_impl
# Default configuration for mysql replication
# replication_strategy = MysqlBinlogReplication
# replication_namespace = trove.guestagent.strategies.replication.mysql_binlog
# replication_user = slave_user
# replication_password = slave_password
# Users to ignore for user create/list/delete operations
# ignore_users = os_admin
# Databases to ignore for db create/list/delete operations
# ignore_dbs = mysql, information_schema, performance_schema
[vertica]
# For vertica, following are the defaults needed:
# mount_point = /var/lib/vertica
# readahead_size = 2048
# guestagent_strategy = trove.common.strategies.cluster.experimental.vertica.guestagent.VerticaGuestAgentStrategy
[redis]
# For redis, the following are the defaults for backup, and restore:
# backup_strategy = RedisBackup
# backup_namespace = trove.guestagent.strategies.backup.experimental.redis_impl
# restore_namespace = trove.guestagent.strategies.restore.experimental.redis_impl
[percona]
backup_namespace = trove.guestagent.strategies.backup.mysql_impl
restore_namespace = trove.guestagent.strategies.restore.mysql_impl
[couchbase]
backup_namespace = trove.guestagent.strategies.backup.experimental.couchbase_impl
restore_namespace = trove.guestagent.strategies.restore.experimental.couchbase_impl
[cassandra]
backup_namespace = trove.guestagent.strategies.backup.experimental.cassandra_impl
restore_namespace = trove.guestagent.strategies.restore.experimental.cassandra_impl
[db2]
# For db2, the following are the defaults for backup, and restore:
# backup_strategy = DB2OfflineBackup
# backup_namespace = trove.guestagent.strategies.backup.experimental.db2_impl
# restore_namespace = trove.guestagent.strategies.restore.experimental.db2_impl
[couchdb]
#For CouchDB, the following are the defaults for backup and restore:
# backup_strategy = CouchDBBackup
# backup_namespace = trove.guestagent.strategies.backup.experimental.couchdb_impl
# restore_namespace = trove.guestagent.strategies.restore.experimental.couchdb_impl

View File

@ -1,19 +0,0 @@
---
version: '2.0'
name: trove
description: Trove Workflows
workflows:
backup_create:
input: [instance, name, description, incremental]
output:
status: <% $.message %>
tasks:
backup_create:
action: trove.backups_create instance=<% $.instance %> name=<% $.name %> description=<% $.description %> incremental=<% $.incremental %>
publish:
message: <% 'Backup complete' %>

View File

@ -1,311 +0,0 @@
[DEFAULT]
# Show debugging output in logs (sets DEBUG log level output)
debug = True
# Address to bind the API server
bind_host = 0.0.0.0
# Port the bind the API server to
bind_port = 8779
# Number of workers for the API service. The default will
# be the number of CPUs available. (integer value)
#trove_api_workers=None
#===================== RPC Configuration =================================
# URL representing the messaging driver to use and its full configuration.
# If not set, we fall back to the 'rpc_backend' option and driver specific
# configuration.
#transport_url=<None>
# The messaging driver to use. Options include rabbit, qpid and zmq.
# Default is rabbit. (string value)
#rpc_backend=rabbit
# The default exchange under which topics are scoped. May be
# overridden by an exchange name specified in the 'transport_url option.
control_exchange = trove
# Maximum line size of message headers to be accepted.
# max_header_line may need to be increased when using large tokens
# (typically those generated by the Keystone v3 API with big service
# catalogs)
# max_header_line = 16384
#DB Api Implementation
db_api_implementation = "trove.db.sqlalchemy.api"
# Configuration options for talking to nova via the novaclient.
trove_auth_url = http://0.0.0.0/identity/v2.0
#nova_compute_url = http://localhost:8774/v2
#cinder_url = http://localhost:8776/v1
#swift_url = http://localhost:8080/v1/AUTH_
#neutron_url = http://localhost:9696/
# nova_compute_url, cinder_url, swift_url, and can all be fetched
# from Keystone. To fetch from Keystone, comment out nova_compute_url,
# cinder_url, swift_url, and and optionally uncomment the lines below.
# Region name of this node. Used when searching catalog. Default value is None.
#os_region_name = RegionOne
# Service type to use when searching catalog.
#nova_compute_service_type = compute
# Service type to use when searching catalog.
#cinder_service_type = volumev2
# Service type to use when searching catalog.
#swift_service_type = object-store
# Service type to use when searching catalog.
#neutron_service_type = network
#ip_regex = ^(15.|123.)
#black_list_regex = ^10.0.0.
# Config options for enabling volume service
trove_volume_support = True
block_device_mapping = vdb
device_path = /dev/vdb
# Maximum volume size for an instance
max_accepted_volume_size = 10
max_instances_per_tenant = 5
# Maximum volume capacity (in GB) spanning across all trove volumes per tenant
max_volumes_per_tenant = 100
max_backups_per_tenant = 5
volume_time_out=30
# Config options for rate limits
http_get_rate = 200
http_post_rate = 200
http_put_rate = 200
http_delete_rate = 200
http_mgmt_post_rate = 200
# Trove DNS
trove_dns_support = False
dns_account_id = 123456
dns_auth_url = http://127.0.0.1/identity/v2.0
dns_username = user
dns_passkey = password
dns_ttl = 3600
dns_domain_name = 'trove.com.'
dns_domain_id = 11111111-1111-1111-1111-111111111111
dns_driver = trove.dns.designate.driver.DesignateDriver
dns_instance_entry_factory = trove.dns.designate.driver.DesignateInstanceEntryFactory
dns_endpoint_url = http://127.0.0.1/v1/
dns_service_type = dns
# Neutron
network_driver = trove.network.nova.NovaNetwork
management_networks =
# Taskmanager queue name
taskmanager_queue = taskmanager
# Auth
admin_roles = admin
# Guest related conf
agent_heartbeat_time = 10
agent_call_low_timeout = 5
agent_call_high_timeout = 150
# Reboot time out for instances
reboot_time_out = 60
# Trove api-paste file name
api_paste_config = api-paste.ini
# ============ Notification System configuration ===========================
# Sets the notification driver used by oslo.messaging. Options include
# messaging, messagingv2, log and routing. Default is 'noop'
# notification_driver=noop
# Topics used for OpenStack notifications, list value. Default is 'notifications'.
# notification_topics=notifications
# ============ Logging information =============================
#log_dir = /integration/report
#log_file = trove-api.log
[database]
# SQLAlchemy connection string for the reference implementation
# registry server. Any valid SQLAlchemy connection string is fine.
# See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
# connection = sqlite:///trove_test.sqlite
connection = mysql+pymysql://root:e1a2c042c828d3566d0a@localhost/trove
#connection = postgresql://trove:trove@localhost/trove
# Period in seconds after which SQLAlchemy should reestablish its connection
# to the database.
#
# MySQL uses a default `wait_timeout` of 8 hours, after which it will drop
# idle connections. This can result in 'MySQL Gone Away' exceptions. If you
# notice this, you can lower this value to ensure that SQLAlchemy reconnects
# before MySQL can drop the connection.
idle_timeout = 3600
# ============ SSL configuration (and enablement) =============================
# In order to enable SSL for the trove api server, uncomment
# the cert_file and key_file - and of course have those files
# accessible. The existence of those setting and files will
# enable SSL.
[profiler]
# If False fully disable profiling feature.
#enabled = False
# If False doesn't trace SQL requests.
#trace_sqlalchemy = True
[ssl]
#cert_file = /path/to/server.crt
#key_file = /path/to/server.key
#optional:
#ca_file = /path/to/ca_file
[oslo_messaging_notifications]
#
# From oslo.messaging
#
# The Driver(s) to handle sending notifications. Possible
# values are messaging, messagingv2, routing, log, test, noop
# (multi valued)
# Deprecated group/name - [DEFAULT]/notification_driver
#driver =
# A URL representing the messaging driver to use for
# notifications. If not set, we fall back to the same
# configuration used for RPC. (string value)
# Deprecated group/name - [DEFAULT]/notification_transport_url
#transport_url = <None>
# AMQP topic used for OpenStack notifications. (list value)
# Deprecated group/name - [rpc_notifier2]/topics
# Deprecated group/name - [DEFAULT]/notification_topics
#topics = notifications
# The maximum number of attempts to re-send a notification
# message which failed to be delivered due to a recoverable
# error. 0 - No retry, -1 - indefinite (integer value)
#retry = -1
[mysql]
root_on_create = False
# Format (single port or port range): A, B-C
# where C greater than B
tcp_ports = 3306
volume_support = True
device_path = /dev/vdb
# Users to ignore for user create/list/delete operations
ignore_users = os_admin, root
ignore_dbs = mysql, information_schema, performance_schema
[redis]
tcp_ports = 6379, 16379
volume_support = True
device_path = /dev/vdb
[cassandra]
tcp_ports = 7000, 7001, 9042, 9160
volume_support = True
device_path = /dev/vdb
[couchbase]
tcp_ports = 8091, 8092, 4369, 11209-11211, 21100-21199
volume_support = True
device_path = /dev/vdb
[mongodb]
tcp_ports = 2500, 27017, 27019
volume_support = True
device_path = /dev/vdb
num_config_servers_per_cluster = 1
num_query_routers_per_cluster = 1
[vertica]
tcp_ports = 5433, 5434, 22, 5444, 5450, 4803
udp_ports = 5433, 4803, 4804, 6453
volume_support = True
device_path = /dev/vdb
cluster_support = True
cluster_member_count = 3
api_strategy = trove.common.strategies.cluster.experimental.vertica.api.VerticaAPIStrategy
# ============ CORS configuration =============================
[cors]
#
# From oslo.middleware.cors
#
# Indicate whether this resource may be shared with the domain received in the
# requests "origin" header. (list value)
#allowed_origin = <None>
# Indicate that the actual request can include user credentials (boolean value)
#allow_credentials = true
# Indicate which headers are safe to expose to the API. Defaults to HTTP Simple
# Headers. (list value)
#expose_headers = X-Auth-Token, X-Subject-Token, X-Service-Token, X-OpenStack-Request-ID
# Maximum cache age of CORS preflight requests. (integer value)
#max_age = 3600
# Indicate which methods can be used during the actual request. (list value)
#allow_methods = GET,PUT,POST,DELETE,PATCH
# Indicate which header field names may be used during the actual request.
# (list value)
#allow_headers = X-Auth-Token, X-Identity-Status, X-Roles, X-Service-Catalog, X-User-Id, X-Tenant-Id, X-OpenStack-Request-ID
[cors.subdomain]
#
# From oslo.middleware.cors
#
# Indicate whether this resource may be shared with the domain received in the
# requests "origin" header. (list value)
#allowed_origin = <None>
# Indicate that the actual request can include user credentials (boolean value)
#allow_credentials = true
# Indicate which headers are safe to expose to the API. Defaults to HTTP Simple
# Headers. (list value)
#expose_headers = X-Auth-Token, X-Subject-Token, X-Service-Token, X-OpenStack-Request-ID
# Maximum cache age of CORS preflight requests. (integer value)
#max_age = 3600
# Indicate which methods can be used during the actual request. (list value)
#allow_methods = GET,PUT,POST,DELETE,PATCH
# Indicate which header field names may be used during the actual request.
# (list value)
#allow_headers = X-Auth-Token, X-Identity-Status, X-Roles, X-Service-Catalog, X-User-Id, X-Tenant-Id, X-OpenStack-Request-ID
[oslo_middleware]
#
# From oslo.middleware
#
# Whether the application is behind a proxy or not. This determines if the
# middleware should parse the headers or not. (boolean value)
#enable_proxy_headers_parsing = false

View File

@ -1,12 +0,0 @@
{
"devstack":null,
"glance":null,
"horizon":null,
"keystone":null,
"nova":null,
"python_openstackclient":null,
"python_novaclient":null,
"trove":null,
"python_troveclient":null,
"tempest":null
}

View File

@ -1,16 +0,0 @@
============
apt-conf-dir
============
This element overrides the default apt.conf.d directory for APT based systems.
Environment Variables
---------------------
DIB_APT_CONF_DIR:
:Required: No
:Default: None
:Description: To override `DIB_APT_CONF_DIR`, set it to the path to your
apt.conf.d. The new apt.conf.d will take effect at build time
and run time.
:Example: ``DIB_APT_CONF_DIR=/etc/apt/apt.conf``

View File

@ -1,21 +0,0 @@
#!/bin/bash
# Override the default /etc/apt/apt.conf.d directory with $DIB_APT_CONF_DIR
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
# exit directly if DIB_APT_CONF_DIR is not defined properly
if [ -z "${DIB_APT_CONF_DIR:-}" ] ; then
echo "DIB_APT_CONF_DIR is not set - no apt.conf.d will be copied in"
exit 0
elif [ ! -d "$DIB_APT_CONF_DIR" ] ; then
echo "$DIB_APT_CONF_DIR is not a valid apt.conf.d directory."
echo "You should assign a proper apt.conf.d directory in DIB_APT_CONF_DIR"
exit 1
fi
# copy the apt.conf to cloudimg
sudo cp -L -f -R $DIB_APT_CONF_DIR $TMP_MOUNT_PATH/etc/apt

View File

@ -4,3 +4,4 @@ pkg-map
source-repositories
svc-map
pip-and-virtualenv
ubuntu-docker

View File

@ -1,34 +0,0 @@
# sometimes the primary key server is unavailable and we should try an
# alternate. see
# https://bugs.launchpad.net/percona-server/+bug/907789. Disable
# shell errexit so we can interrogate the exit code and take action
# based on the exit code. We will reenable it later.
#
# NOTE(zhaochao): we still have this problem from time to time, so it's
# better use more reliable keyservers and just retry on that(for now, 3
# tries should be fine).
# According to:
# [1] https://www.gnupg.org/faq/gnupg-faq.html#new_user_default_keyserver
# [2] https://sks-keyservers.net/overview-of-pools.php
# we'll just the primary suggested pool: pool.sks-keyservers.net.
function get_key_robust() {
KEY=$1
set +e
tries=1
while [ $tries -le 3 ]; do
if [ $tries -eq 3 ]; then
set -e
fi
echo "Importing the key, try: $tries"
# Behind a firewall should use the port 80 instead of the default port 11371
apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys ${KEY} && break
tries=$((tries+1))
done
set -e
}
export -f get_key_robust

View File

@ -0,0 +1,51 @@
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
DEV_MODE=${DEV_MODE:-"true"}
SCRIPTDIR=$(dirname $0)
GUEST_USERNAME=${GUEST_USERNAME:-"ubuntu"}
GUEST_VENV=/opt/guest-agent-venv
for folder in "/var/lib/trove" "/etc/trove" "/etc/trove/certs" "/etc/trove/conf.d" "/var/log/trove" "/opt/trove-guestagent"; do
mkdir -p ${folder}
chown -R ${GUEST_USERNAME}:root ${folder}
done
install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.logrotate /etc/logrotate.d/guest-agent
# Create a virtual environment (with dependencies installed) for guest agent service
${DIB_PYTHON} -m virtualenv ${GUEST_VENV}
${GUEST_VENV}/bin/pip install pip --upgrade
${GUEST_VENV}/bin/pip install -U -c /opt/upper-constraints.txt /opt/guest-agent
chown -R ${GUEST_USERNAME}:root ${GUEST_VENV}
if [[ ${DEV_MODE} == "true" ]]; then
[[ -n "${HOST_SCP_USERNAME}" ]] || die "HOST_SCP_USERNAME needs to be set to the trovestack host user"
[[ -n "${ESCAPED_PATH_TROVE}" ]] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host"
sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${SCRIPTDIR}/guest-agent-dev.service > /etc/systemd/system/guest-agent.service
else
# Link the trove-guestagent out to /usr/local/bin where the startup scripts look for
ln -s ${GUEST_VENV}/bin/trove-guestagent /usr/local/bin/guest-agent || true
case "$DIB_INIT_SYSTEM" in
systemd)
sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g" ${SCRIPTDIR}/guest-agent.service > /etc/systemd/system/guest-agent.service
;;
upstart)
install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.conf /etc/init/guest-agent.conf
;;
sysv)
install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.init /etc/init.d/guest-agent.init
;;
*)
echo "Unsupported init system"
exit 1
;;
esac
fi

View File

@ -1,45 +0,0 @@
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
SCRIPTDIR=$(dirname $0)
GUEST_VENV=/opt/guest-agent-venv
GUEST_USERNAME=${GUEST_USERNAME:-"ubuntu"}
# Create a virtual environment for guest agent
${DIB_PYTHON} -m virtualenv ${GUEST_VENV}
${GUEST_VENV}/bin/pip install pip --upgrade
${GUEST_VENV}/bin/pip install -U -c /opt/upper-constraints.txt /opt/guest-agent
chown -R ${GUEST_USERNAME}:root ${GUEST_VENV}
# Link the trove-guestagent out to /usr/local/bin where the startup scripts look for
ln -s ${GUEST_VENV}/bin/trove-guestagent /usr/local/bin/guest-agent || true
for folder in "/var/lib/trove" "/etc/trove" "/etc/trove/certs" "/etc/trove/conf.d" "/var/log/trove"; do
mkdir -p ${folder}
chown -R ${GUEST_USERNAME}:root ${folder}
done
install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.logrotate /etc/logrotate.d/guest-agent
case "$DIB_INIT_SYSTEM" in
systemd)
mkdir -p /usr/lib/systemd/system
touch /usr/lib/systemd/system/guest-agent.service
sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g" ${SCRIPTDIR}/guest-agent.service > /usr/lib/systemd/system/guest-agent.service
;;
upstart)
install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.conf /etc/init/guest-agent.conf
;;
sysv)
install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.init /etc/init.d/guest-agent.init
;;
*)
echo "Unsupported init system"
exit 1
;;
esac

View File

@ -0,0 +1,31 @@
[Unit]
Description=OpenStack Trove Guest Agent Service for Development
After=syslog.target network.target
[Install]
WantedBy=multi-user.target
[Service]
Type=simple
User=GUEST_USERNAME
Group=GUEST_USERNAME
# This script is only for testing purpose for dev_mode=true, the controller
# IP address should be defined in /etc/trove/controller.conf, e.g.
# CONTROLLER=192.168.32.151
EnvironmentFile=/etc/trove/controller.conf
# If ~/trove-installed does not exist, copy the trove source from
# the user's development environment, then touch the sentinel file
ExecStartPre=/bin/bash -c "test -e /home/GUEST_USERNAME/trove-installed || sudo rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i /home/GUEST_USERNAME/.ssh/id_rsa' -az --exclude='.*' HOST_SCP_USERNAME@$CONTROLLER:PATH_TROVE/ /home/GUEST_USERNAME/trove && touch /home/GUEST_USERNAME/trove-installed"
ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove /var/log/trove/ /home/GUEST_USERNAME/trove"
# Take care of the changes in requirements.txt
ExecStartPre=/bin/bash -c "sudo /opt/guest-agent-venv/bin/pip install -r /home/GUEST_USERNAME/trove/requirements.txt -c /opt/upper-constraints.txt"
# Start guest-agent.service in virtual environment
ExecStart=/bin/bash -c "/opt/guest-agent-venv/bin/python /home/GUEST_USERNAME/trove/contrib/trove-guestagent --config-dir=/etc/trove/conf.d"
TimeoutSec=300
Restart=on-failure

View File

@ -1,15 +1,16 @@
[Unit]
Description=OpenStack Trove Guest Agent
Description=OpenStack Trove Guest Agent Service
After=network.target syslog.service
Wants=syslog.service
[Service]
User=GUEST_USERNAME
Group=GUEST_USERNAME
ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove/conf.d"
ExecStart=/usr/local/bin/guest-agent --config-dir=/etc/trove/conf.d
KillMode=mixed
Restart=always
ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove /var/log/trove/"
ExecStart=/usr/local/bin/guest-agent --config-dir=/etc/trove/conf.d
[Install]
WantedBy=multi-user.target

View File

@ -1,53 +1,15 @@
guest-agent:
installtype: package
build-essential:
installtype: source
python3-all:
python3-all-dev:
python3-pip:
python3-sqlalchemy:
libxml2-dev:
libxslt1-dev:
libffi-dev:
installtype: source
libssl-dev:
installtype: source
python-dev:
installtype: source
acl:
acpid:
apparmor:
apparmor-utils:
apt-transport-https:
at:
bash-completion:
cloud-guest-utils:
cloud-init:
cron:
curl:
dbus:
dkms:
dmeventd:
ethtool:
gawk:
ifenslave:
ifupdown:
iptables:
iputils-tracepath:
irqbalance:
isc-dhcp-client:
less:
logrotate:
lsof:
net-tools:
netbase:
netcat-openbsd:
open-vm-tools:
arch: i386, amd64
libyaml-dev:
openssh-client:
openssh-server:
pollinate:
psmisc:
rsyslog:
socat:
tcpdump:
ubuntu-cloudimage-keyring:
ureadahead:
uuid-runtime:
vim-tiny:
vlan:
rsync:

View File

@ -1,8 +0,0 @@
This element clears out /etc/resolv.conf and prevents dhclient from populating
it with data from DHCP. This means that DNS resolution will not work from the
guest. This is OK because all outbound connections from the guest will
be based using raw IP addresses.
In addition we remove dns from the nsswitch.conf hosts setting.
This means that the guest never waits for DNS timeouts to occur.

View File

@ -1,19 +0,0 @@
#!/bin/bash
echo "" > /etc/resolv.conf
echo "" > /etc/resolv.conf.ORIG
if [ -d /etc/dhcp/dhclient-enter-hooks.d ]; then
# Debian/Ubuntu
echo "#!/bin/sh
make_resolv_conf() { : ; }" > /etc/dhcp/dhclient-enter-hooks.d/noresolvconf
chmod +x /etc/dhcp/dhclient-enter-hooks.d/noresolvconf
rm -f /etc/dhcp/dhclient-enter-hooks.d/resolvconf
else
# RHEL/CentOS/Fedora
echo "#!/bin/sh
make_resolv_conf() { : ; }" > /etc/dhclient-enter-hooks
chmod +x /etc/dhclient-enter-hooks
fi
if [ -e /etc/nsswitch.conf ]; then
sed -i -e "/hosts:/ s/dns//g" /etc/nsswitch.conf
fi

View File

@ -0,0 +1 @@
ubuntu-guest

View File

@ -0,0 +1,19 @@
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
GUEST_USERNAME=${GUEST_USERNAME:-"ubuntu"}
echo "Installing docker"
export DEBIAN_FRONTEND=noninteractive
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu ${DIB_RELEASE} stable"
apt-get update
apt-get install -y -qq docker-ce >/dev/null
echo "Adding ${GUEST_USERNAME} user to docker group"
usermod -aG docker ${GUEST_USERNAME}

View File

@ -1,34 +0,0 @@
# sometimes the primary key server is unavailable and we should try an
# alternate. see
# https://bugs.launchpad.net/percona-server/+bug/907789. Disable
# shell errexit so we can interrogate the exit code and take action
# based on the exit code. We will reenable it later.
#
# NOTE(zhaochao): we still have this problem from time to time, so it's
# better use more reliable keyservers and just retry on that(for now, 3
# tries should be fine).
# According to:
# [1] https://www.gnupg.org/faq/gnupg-faq.html#new_user_default_keyserver
# [2] https://sks-keyservers.net/overview-of-pools.php
# we'll just the primary suggested pool: pool.sks-keyservers.net.
function get_key_robust() {
KEY=$1
set +e
tries=1
while [ $tries -le 3 ]; do
if [ $tries -eq 3 ]; then
set -e
fi
echo "Importing the key, try: $tries"
# Behind a firewall should use the port 80 instead of the default port 11371
apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys ${KEY} && break
tries=$((tries+1))
done
set -e
}
export -f get_key_robust

View File

@ -0,0 +1,17 @@
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
source $_LIB/die
[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
# Guest agent needs to ssh into the controller to download code in dev mode.
if [[ ${DEV_MODE} == "true" && -e ${SSH_DIR}/id_rsa ]]; then
sudo -Hiu ${HOST_SCP_USERNAME} dd if=${SSH_DIR}/id_rsa of=${TMP_HOOKS_PATH}/id_rsa
sudo -Hiu ${HOST_SCP_USERNAME} dd if=${SSH_DIR}/id_rsa.pub of=${TMP_HOOKS_PATH}/id_rsa.pub
fi

View File

@ -1,28 +0,0 @@
#!/bin/bash
set -e
set -o xtrace
# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
# PURPOSE: Setup the requirements file for use by 15-reddwarf-dep
source $_LIB/die
TROVE_BRANCH=${TROVE_BRANCH:-'master'}
REQUIREMENTS_FILE=${TROVESTACK_SCRIPTS}/../../requirements.txt
[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
[ -e ${REQUIREMENTS_FILE} ] || die "Requirements not found"
[ -n "$HOST_USERNAME" ] || die "HOST_USERNAME not set"
sudo -Hiu ${HOST_USERNAME} dd if=${REQUIREMENTS_FILE} of=${TMP_HOOKS_PATH}/requirements.txt
UC_FILE=upper-constraints.txt
UC_DIR=$(pwd)
UC_BRANCH=${TROVE_BRANCH##stable/}
curl -L -o "${UC_DIR}/${UC_FILE}" "https://releases.openstack.org/constraints/upper/${UC_BRANCH}"
if [ -f "${UC_DIR}/${UC_FILE}" ]; then
sudo -Hiu ${HOST_USERNAME} dd if="${UC_DIR}/${UC_FILE}" of=${TMP_HOOKS_PATH}/${UC_FILE}
rm -f "${UC_DIR}/${UC_FILE}"
fi

View File

@ -1,27 +0,0 @@
#!/bin/bash
set -e
set -o xtrace
# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
# PURPOSE: creates the SSH key on the host if it doesn't exist. Then this copies the keys over to a staging area where
# they will be duplicated in the guest VM.
# This process allows the host to log into the guest but more importantly the guest phones home to get the trove
# source
source $_LIB/die
[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
[ -n "${HOST_USERNAME}" ] || die "HOST_USERNAME needs to be set to the user for the current user on the host"
if [ `whoami` = "root" ]; then
die "This should not be run as root"
fi
# Guest agent needs to ssh into the controller to download code in dev mode.
if [ -e ${SSH_DIR}/id_rsa ]; then
sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa of=${TMP_HOOKS_PATH}/id_rsa
sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa.pub of=${TMP_HOOKS_PATH}/id_rsa.pub
else
die "SSH keys must exist"
fi

View File

@ -1,10 +0,0 @@
#!/bin/bash
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Install basic services and applications
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
apt-get --allow-unauthenticated -y install ntp apparmor-utils

View File

@ -0,0 +1,22 @@
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
GUEST_SSH_DIR="/home/${GUEST_USERNAME}/.ssh"
TMP_HOOKS_DIR="/tmp/in_target.d"
if [ ! -e ${GUEST_SSH_DIR} ]; then
# this method worked more reliable in vmware fusion over doing sudo -Hiu ${GUEST_USERNAME}
mkdir ${GUEST_SSH_DIR}
chown -R ${GUEST_USERNAME}:${GUEST_USERNAME} ${GUEST_SSH_DIR}
fi
if [[ ${DEV_MODE} == "true" && -e "${TMP_HOOKS_DIR}/id_rsa" ]]; then
sudo -Hiu ${GUEST_USERNAME} dd of=${GUEST_SSH_DIR}/id_rsa.pub if=${TMP_HOOKS_DIR}/id_rsa.pub
sudo -Hiu ${GUEST_USERNAME} dd of=${GUEST_SSH_DIR}/id_rsa if=${TMP_HOOKS_DIR}/id_rsa
sudo -Hiu ${GUEST_USERNAME} chmod 600 ${GUEST_SSH_DIR}/id_rsa
fi

View File

@ -1,37 +0,0 @@
#!/bin/bash
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Install trove guest python dependencies - see trovestack functions_qemu
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
apt-get --allow-unauthenticated -y install \
libxml2-dev libxslt1-dev libffi-dev libssl-dev libyaml-dev \
python3-pip python3-sqlalchemy python3-setuptools
# Install python 3.7, some python lib (e.g. oslo.concurrency>4.0.0) requries
# Python 3.7
add-apt-repository --yes ppa:deadsnakes/ppa
apt update
apt install -y python3.7 python3.7-dev
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 10
python3.5 -m pip install pip==9.0.3
python3.5 -m pip install -U wheel setuptools
TMP_HOOKS_DIR="/tmp/in_target.d"
UPPER_CONSTRAINTS=
if [ -f ${TMP_HOOKS_DIR}/upper-constraints.txt ]; then
UPPER_CONSTRAINTS=" -c ${TMP_HOOKS_DIR}/upper-constraints.txt"
fi
python3.7 -m pip install pip==9.0.3
python3.7 -m pip install -U wheel setuptools
python3.7 -m pip install --upgrade -r ${TMP_HOOKS_DIR}/requirements.txt ${UPPER_CONSTRAINTS}
echo "diagnostic pip freeze output follows"
python3.7 -m pip freeze
echo "diagnostic pip freeze output above"

View File

@ -1,18 +0,0 @@
#!/bin/bash
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Add the guest image user that will own the trove agent source...if the user does not already exist
set -e
set -o xtrace
if ! id -u ${GUEST_USERNAME} >/dev/null 2>&1; then
echo "Adding ${GUEST_USERNAME} user"
useradd -G sudo -m ${GUEST_USERNAME} -s /bin/bash
chown ${GUEST_USERNAME}:${GUEST_USERNAME} /home/${GUEST_USERNAME}
passwd ${GUEST_USERNAME} <<_EOF_
${GUEST_USERNAME}
${GUEST_USERNAME}
_EOF_
fi

View File

@ -1,25 +0,0 @@
#!/bin/bash
# PURPOSE: take "staged" ssh keys (see extra-data.d/62-ssh-key) and put them in the GUEST_USERS home directory
# In future, this should be removed and use Nova keypair to inject ssh keys.
set -e
set -o xtrace
SSH_DIR="/home/${GUEST_USERNAME}/.ssh"
TMP_HOOKS_DIR="/tmp/in_target.d"
if [ ! -e ${SSH_DIR} ]; then
# this method worked more reliable in vmware fusion over doing sudo -Hiu ${GUEST_USERNAME}
mkdir ${SSH_DIR}
chown ${GUEST_USERNAME}:${GUEST_USERNAME} ${SSH_DIR}
fi
if [ -e "${TMP_HOOKS_DIR}/id_rsa" ]; then
sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa.pub if=${TMP_HOOKS_DIR}/id_rsa.pub
sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa if=${TMP_HOOKS_DIR}/id_rsa
sudo -Hiu ${GUEST_USERNAME} chmod 600 ${SSH_DIR}/id_rsa
else
echo "SSH Keys were not staged by host"
exit -1
fi

View File

@ -1,8 +0,0 @@
#!/bin/bash
# Regenerate host keys now. XXX: Really should be a cloud-init task, should get
# that working.
set -e
set -o xtrace
dpkg-reconfigure openssh-server

View File

@ -1,11 +0,0 @@
#!/bin/sh
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Delete contents of apt cache on guest (saves image disk space)
set -e
set -o xtrace
apt-get clean

View File

@ -1,7 +0,0 @@
#!/bin/bash
# Install baseline packages and tools.
set -e
set -o xtrace
apt-get --allow-unauthenticated install -y language-pack-en python-software-properties software-properties-common

View File

@ -0,0 +1,7 @@
#!/bin/bash
# Install baseline packages and tools.
set -e
set -o xtrace
DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated install -y -qq software-properties-common apt-transport-https ca-certificates ntp >/dev/null

View File

@ -1,3 +0,0 @@
Sets up a MariaDB server install in the image.
TODO: auto-tune settings based on host resources or metadata service.

View File

@ -1,11 +0,0 @@
#!/bin/sh
set -e
#CONTEXT: chroot on host
#PURPOSE: Allows mysqld to create temporary files when restoring backups
cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld
/tmp/ rw,
/tmp/** rwk,
EOF

View File

@ -1,12 +0,0 @@
#!/bin/sh
set -e
#CONTEXT: chroot on host
#PURPOSE: Allows mysqld to create temporary files when restoring backups
mkdir -p /etc/apparmor.d/local/
cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld
/tmp/ rw,
/tmp/** rwk,
EOF

View File

@ -1,25 +0,0 @@
#!/bin/sh
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
cat > "/etc/sysctl.d/10-postgresql-performance.conf" << _EOF_
# See 'http://www.postgresql.org/docs/9.6/static/kernel-resources.html'
# for best practices.
# It is recommended to disable memory overcommit,
# but the Python interpreter may require it on smaller flavors.
# We therefore stick with the heuristic overcommit setting.
vm.overcommit_memory=0
_EOF_
apt-get --allow-unauthenticated -y install libpq-dev postgresql-12 postgresql-server-dev-12 postgresql-client-12
pgsql_conf=/etc/postgresql/12/main/postgresql.conf
sed -i "/listen_addresses/c listen_addresses = '*'" ${pgsql_conf}
systemctl restart postgresql
# Install the native Python client.
pip3 install psycopg2

View File

@ -1,14 +0,0 @@
#!/bin/sh
set -e
set -o xtrace
[ -n "${DIB_RELEASE}" ] || die "RELEASE must be set to a valid Ubuntu release (e.g. trusty)"
cat <<EOL > /etc/apt/sources.list.d/postgresql.list
deb http://apt.postgresql.org/pub/repos/apt/ ${DIB_RELEASE}-pgdg main
EOL
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
apt-get update

View File

@ -1,21 +0,0 @@
#!/bin/bash
set -e
set -o xtrace
# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
# PURPOSE: stages the bootstrap file and upstart conf file while replacing variables so that guest image is properly
# configured
source $_LIB/die
[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance"
[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host"
[ -n "${TROVESTACK_SCRIPTS}" ] || die "TROVESTACK_SCRIPTS needs to be set to the trove/integration/scripts dir"
[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir"
sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${TROVESTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service

View File

@ -1,8 +0,0 @@
#!/bin/sh
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: take "staged" trove-guest.conf file and put it in the init directory on guest image
dd if=/tmp/in_target.d/trove-guest.service of=/etc/systemd/system/trove-guest.service
systemctl enable trove-guest.service

View File

@ -1,12 +0,0 @@
#!/bin/sh
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: configure trove-guest service to use system store of trusted certificates
GUEST_UNIT_DROPINS="/etc/systemd/system/trove-guest.service.d"
mkdir -v -p ${GUEST_UNIT_DROPINS}
cat <<EOF > ${GUEST_UNIT_DROPINS}/30-use-system-certificates.conf
[Service]
Environment=REQUESTS_CA_BUNDLE=/etc/ssl/certs
EOF

View File

@ -1,29 +0,0 @@
#!/bin/bash
set -e
set -o xtrace
# The HWE stack must be installed for nested virtualization on ppc64el. This
# environment variable is set automatically by trovestack, but it may also be
# set by the user when manually invoking disk-image-create.
case "$DIB_USE_HWE_KERNEL" in
true|True|TRUE|yes|Yes|YES)
DIB_USE_HWE_KERNEL=true
;;
*)
DIB_USE_HWE_KERNEL=false
;;
esac
if [ "$DIB_USE_HWE_KERNEL" == "true" ]; then
export DEBIAN_FRONTEND=noninteractive
PKG_ARCH=$(dpkg --print-architecture)
case "$PKG_ARCH" in
amd64|arm64|ppc64el|s390x)
apt-get --allow-unauthenticated install -y linux-generic-hwe-16.04
;;
esac
fi

View File

@ -1,90 +0,0 @@
#!/bin/bash
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Install basic services and applications
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
apt-get -y purge acpid\
apport\
apport-symptoms\
apt-transport-https\
aptitude\
at\
bash-completion\
bc\
bind9-host\
bsdmainutils\
busybox-static\
byobu\
command-not-found\
command-not-found-data\
curl\
dbus\
dmidecode\
dosfstools\
ed\
fonts-ubuntu-font-family-console\
friendly-recovery\
ftp\
fuse\
geoip-database\
groff-base\
hdparm\
info\
install-info\
iptables\
iputils-tracepath\
irqbalance\
language-selector-common\
libaccountsservice0\
libevent-2.0-5\
libgeoip1\
libnfnetlink0\
libpcap0.8\
libpci3\
libpipeline1\
libpolkit-gobject-1-0\
libsasl2-modules\
libusb-1.0-0\
lshw\
lsof\
ltrace\
man-db\
mlocate\
mtr-tiny\
nano\
ntfs-3g\
parted\
patch\
plymouth-theme-ubuntu-text\
popularity-contest\
powermgmt-base\
ppp\
screen\
shared-mime-info\
strace\
tcpdump\
telnet\
time\
tmux\
ubuntu-standard\
ufw\
update-manager-core\
update-notifier-common\
usbutils\
uuid-runtime\
# The following packages cannot be removed as they cause cloud-init to be
# uninstalled in Ubuntu 14.04
# gir1.2-glib-2.0
# libdbus-glib-1-2
# libgirepository-1.0-1
# python-chardet
# python-serial
# xz-utils
apt-get -y autoremove

View File

@ -1,39 +0,0 @@
#!/bin/bash
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Install controller base required packages
# Refer to https://mariadb.com/kb/en/library/installing-mariadb-deb-files
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
# These GPG key IDs are used to fetch keys from a keyserver on Ubuntu & Debian
apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
curl -sS https://downloads.mariadb.com/MariaDB/mariadb_repo_setup |
bash -s -- --mariadb-server-version="mariadb-10.4" --skip-key-import --skip-maxscale
apt-get install -y -qq apt-transport-https ca-certificates gnupg2
# NOTE(lxkong): Refer to https://www.percona.com/doc/percona-xtrabackup/2.4/installation/apt_repo.html
wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
# Disable password prompt
debconf-set-selections <<< "mariadb-server mysql-server/root_password password ''"
debconf-set-selections <<< "mariadb-server mysql-server/root_password_again password ''"
apt-get update -qq
apt-get install -y -qq --allow-unauthenticated mariadb-server mariadb-client galera-4 libmariadb3 mariadb-backup mariadb-common
cat <<EOF >/etc/mysql/conf.d/no_perf_schema.cnf
[mysqld]
performance_schema = off
EOF
chown mysql:mysql /etc/mysql/my.cnf
rm -f /etc/init.d/mysql
systemctl daemon-reload
systemctl enable mariadb

View File

@ -1,39 +0,0 @@
#!/bin/bash
# CONTEXT: GUEST during CONSTRUCTION as ROOT
# PURPOSE: Install controller base required packages
set -e
set -o xtrace
export DEBIAN_FRONTEND=noninteractive
apt-get --allow-unauthenticated -y install mysql-client mysql-server gnupg2
# NOTE(lxkong): Refer to https://www.percona.com/doc/percona-xtrabackup/2.4/installation/apt_repo.html
wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
apt-get update
# Xenial provides mysql 5.7 which requires percona-xtrabackup-24
PXB_VERSION_OVERRIDE=24
apt-get --allow-unauthenticated -y install percona-xtrabackup-${PXB_VERSION_OVERRIDE}
cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_
[mysqld]
performance_schema = off
show_compatibility_56 = on
_EOF_
mv /etc/mysql/my.cnf.fallback /etc/mysql/my.cnf
chown mysql:mysql /etc/mysql/my.cnf
cat >/etc/mysql/my.cnf <<_EOF_
[mysql]
!includedir /etc/mysql/conf.d/
_EOF_
if [ -e /etc/init/mysql.conf ]; then
rm -f /etc/init/mysql.conf
fi
systemctl enable mysql

View File

@ -3,44 +3,36 @@
# Additional functions that would mostly just pertain to a Ubuntu + Qemu setup
#
function build_vm() {
exclaim "Actually building the image, this can take up to 15 minutes"
rm -rf ~/.cache/image-create
function build_guest_image() {
exclaim "Actually building the image, params: $@"
local datastore_type=$1
local guest_os=$2
local guest_release=$3
local dev_mode=$4
local guest_username=$5
local image_output=$6
local guest_os=$1
local guest_release=$2
local dev_mode=$3
local guest_username=$4
local image_output=$5
local elementes="base vm"
local trove_elements_path=${PATH_TROVE}/integration/scripts/files/elements
local GUEST_IMAGETYPE=${GUEST_IMAGETYPE:-"qcow2"}
local GUEST_IMAGESIZE=${GUEST_IMAGESIZE:-4}
local GUEST_IMAGESIZE=${GUEST_IMAGESIZE:-3}
local GUEST_CACHEDIR=${GUEST_CACHEDIR:-"$HOME/.cache/image-create"}
rm -rf ${GUEST_CACHEDIR}
local working_dir=$(dirname ${image_output})
export GUEST_USERNAME=${guest_username}
export HOST_SCP_USERNAME=${HOST_SCP_USERNAME:-$(whoami)}
export ESCAPED_PATH_TROVE=$(echo ${PATH_TROVE} | sed 's/\//\\\//g')
export DEV_MODE=${dev_mode,,}
# In dev mode, the trove guest agent needs to download trove code from
# trove-taskmanager host during service initialization.
if [[ "${dev_mode,,}" == "true" ]]; then
export PATH_TROVE=${PATH_TROVE}
export ESCAPED_PATH_TROVE=$(echo ${PATH_TROVE} | sed 's/\//\\\//g')
export GUEST_LOGDIR=${GUEST_LOGDIR:-"/var/log/trove/"}
export ESCAPED_GUEST_LOGDIR=$(echo ${GUEST_LOGDIR} | sed 's/\//\\\//g')
export TROVESTACK_SCRIPTS=${TROVESTACK_SCRIPTS}
export HOST_SCP_USERNAME=${HOST_SCP_USERNAME:-$(whoami)}
export HOST_USERNAME=${HOST_SCP_USERNAME}
if [[ "${DEV_MODE}" == "true" ]]; then
export SSH_DIR=${SSH_DIR:-"$HOME/.ssh"}
export DEST=${DEST:-'/opt/stack'}
export TROVE_BRANCH=${TROVE_BRANCH:-'master'}
manage_ssh_keys
fi
# For system-wide installs, DIB will automatically find the elements, so we only check local path
if [ "${DIB_LOCAL_ELEMENTS_PATH}" ]; then
if [[ "${DIB_LOCAL_ELEMENTS_PATH}" ]]; then
export ELEMENTS_PATH=${trove_elements_path}:${DIB_LOCAL_ELEMENTS_PATH}
else
export ELEMENTS_PATH=${trove_elements_path}
@ -51,35 +43,25 @@ function build_vm() {
# https://cloud-images.ubuntu.com/releases is more stable than the daily
# builds (https://cloud-images.ubuntu.com/xenial/current/),
# e.g. sometimes SHA256SUMS file is missing in the daily builds
declare -A releasemapping=( ["xenial"]="16.04" ["bionic"]="18.04")
# e.g. sometimes SHA256SUMS file is missing in the daily builds website.
# Ref: diskimage_builder/elements/ubuntu/root.d/10-cache-ubuntu-tarball
declare -A image_file_mapping=( ["xenial"]="ubuntu-16.04-server-cloudimg-amd64-root.tar.gz" ["bionic"]="ubuntu-18.04-server-cloudimg-amd64.squashfs" )
export DIB_CLOUD_IMAGES="https://cloud-images.ubuntu.com/releases/${DIB_RELEASE}/release/"
export BASE_IMAGE_FILE="ubuntu-${releasemapping[${DIB_RELEASE}]}-server-cloudimg-amd64-root.tar.gz"
export BASE_IMAGE_FILE=${image_file_mapping[${DIB_RELEASE}]}
TEMP=$(mktemp -d ${working_dir}/diskimage-create.XXXXXXX)
pushd $TEMP > /dev/null
elementes="$elementes ${guest_os}"
if [[ "${dev_mode,,}" == "false" ]]; then
elementes="$elementes pip-and-virtualenv"
elementes="$elementes pip-cache"
elementes="$elementes guest-agent"
else
# Install guest agent dependencies, user, etc.
elementes="$elementes ${guest_os}-guest"
# Install guest agent service
elementes="$elementes ${guest_os}-${guest_release}-guest"
fi
elementes="$elementes ${guest_os}-${datastore_type}"
elementes="$elementes ${guest_os}-${guest_release}-${datastore_type}"
elementes="$elementes ${guest_os}-docker"
# Build the image
disk-image-create -x \
-a amd64 \
-o ${image_output} \
-t ${GUEST_IMAGETYPE} \
--image-size ${GUEST_IMAGESIZE} \
--image-cache ${GUEST_CACHEDIR} \
$elementes
@ -91,25 +73,6 @@ function build_vm() {
exclaim "Image ${image_output} was built successfully."
}
function build_guest_image() {
exclaim "Params for build_guest_image function: $@"
local datastore_type=${1:-"mysql"}
local guest_os=${2:-"ubuntu"}
local guest_release=${3:-"xenial"}
local dev_mode=${4:-"true"}
local guest_username=${5:-"ubuntu"}
local output=$6
VALID_SERVICES='mysql percona mariadb redis cassandra couchbase mongodb postgresql couchdb vertica db2 pxc'
if ! [[ " $VALID_SERVICES " =~ " $datastore_type " ]]; then
exclaim "You did not pass in a valid datastore type. Valid types are:" $VALID_SERVICES
exit 1
fi
build_vm ${datastore_type} ${guest_os} ${guest_release} ${dev_mode} ${guest_username} ${output}
}
function clean_instances() {
LIST=`virsh -q list|awk '{print $1}'`
for i in $LIST; do sudo virsh destroy $i; done
@ -117,6 +80,8 @@ function clean_instances() {
# In dev mode, guest agent needs to ssh into the controller to download code.
function manage_ssh_keys() {
SSH_DIR=${SSH_DIR:-"$HOME/.ssh"}
if [ -d ${SSH_DIR} ]; then
echo "${SSH_DIR} already exists"
else

View File

@ -1,3 +0,0 @@
[[post-config|\$CINDER_CONF]]
[DEFAULT]
notification_driver = messagingv2

View File

@ -1,3 +0,0 @@
[[post-config|\$NOVA_CONF]]
[DEFAULT]
instance_usage_audit = True

View File

@ -1,3 +0,0 @@
[[post-config|\$CEILOMETER_CONF]]
[notification]
store_events = True

View File

@ -1,42 +0,0 @@
#
# Files in this directory are automatically added to the devstack
# local.conf file, between a specific set of tags.
#
# Filenames must end with '.rc' to be recognized; sample.rc is
# ignored.
#
# A '\' is required in front of any devstack variables since all
# .rc files are parsed first (using eval).
#
# Meta section headings must be included in each file, such as:
# [[local|localrc]]
# as the order of inserting the files is not guaranteed.
#
# All files are inherently included by default - to exclude a file,
# add a variable 'FILENAME_IN_UPPERCASE_MINUS_RC=false' in trovestack.rc
# For Example: USING_VAGRANT=false (for the using_vagrant.rc file).
#
# Symbolic links are followed, so additional files can be loaded
# by placing them in an external directory and linking it in
# local.conf.d (this should allow complete flexibility in setting
# up testing options).
# For Example:
# cd /path/to/trove/integration/scripts/local.conf.d
# ln -s $HOME/local.conf.d local.conf.d
# cp /path/to/my_conf.rc $HOME/local.conf.d
[[local|localrc]]
# Put regular devstack variables under this meta section heading.
# This section is written out to a file and sourced by devstack,
# so it can contain logic as well.
# The following section types should only contain ini file style
# section headings and name=value pairs
[[post-config|\$TROVE_CONF]]
[[post-config|\$TROVE_TASKMANAGER_CONF]]
[[post-config|\$TROVE_CONDUCTOR_CONF]]
[[post-config|\$TROVE_API_PASTE_INI]]

View File

@ -1,24 +0,0 @@
[[post-config|\$TROVE_CONF]]
[profiler]
enabled = $ENABLE_PROFILER
trace_sqlalchemy = $PROFILER_TRACE_SQL
[[post-config|\$TROVE_TASKMANAGER_CONF]]
[profiler]
enabled = $ENABLE_PROFILER
trace_sqlalchemy = $PROFILER_TRACE_SQL
[[post-config|\$TROVE_CONDUCTOR_CONF]]
[profiler]
enabled = $ENABLE_PROFILER
trace_sqlalchemy = $PROFILER_TRACE_SQL
[[post-config|\$TROVE_GUESTAGENT_CONF]]
[profiler]
enabled = $ENABLE_PROFILER
trace_sqlalchemy = $PROFILER_TRACE_SQL
[[post-config|\$TROVE_API_PASTE_INI]]
[filter:osprofiler]
enabled = $ENABLE_PROFILER
hmac_keys = $PROFILER_HMAC_KEYS

View File

@ -1,4 +0,0 @@
[[local|localrc]]
# force kvm as the libvirt type.
LIBVIRT_TYPE=kvm

View File

@ -1,3 +0,0 @@
[[local|localrc]]
KEYSTONE_TOKEN_FORMAT=UUID

View File

@ -1,9 +0,0 @@
[[local|localrc]]
# This is similar to code found at
# https://github.com/bcwaldon/vagrant_devstack/blob/master/Vagrantfile
# and seems to make instances ping'able in VirtualBox.
FLAT_INTERFACE=eth1
PUBLIC_INTERFACE=eth1
FLOATING_RANGE=`ip_chunk eth0 1`.`ip_chunk eth0 2`.`ip_chunk eth0 3`.128/28
HOST_IP=`ip_chunk eth0 1`.`ip_chunk eth0 2`.`ip_chunk eth0 3`.`ip_chunk eth0 4`

View File

@ -1,37 +0,0 @@
$TROVE_PRESENT_TAG
# Set some arguments for devstack.
#
# Note: This file contains autogenerated parts.
# All lines are removed from between the tag/end of tag
# markers (lines with '$MARKER_TOKEN' at beginning and end) and
# are replaced by trovestack.
# Edits to these sections will not persist.
#
# See the '$USER_OPTS_TAG' section
# for ways to insert user args into this file.
#
#
# This section is for things that belong in localrc
# It comes from $DEFAULT_LOCALRC
#
[[local|localrc]]
$LOCALRC_OPTS_TAG
$LOCALRC_OPTS_TAG_END
#
# User options here were inserted from the file USER_LOCAL_CONF
# (defaults to $USERHOME/.$LOCAL_CONF)
#
$USER_OPTS_TAG
$USER_OPTS_TAG_END
#
# Additional options here were inserted by trovestack
# automatically from files in $LOCAL_CONF_D
#
$ADD_OPTS_TAG
$ADD_OPTS_TAG_END

View File

@ -124,7 +124,7 @@ if is_fedora; then
else
PKG_INSTALL_OPTS="DEBIAN_FRONTEND=noninteractive"
PKG_MGR=apt-get
PKG_GET_ARGS="-y --allow-unauthenticated --force-yes"
PKG_GET_ARGS="-y --allow-unauthenticated --force-yes -qq"
fi
PKG_INSTALL_ARG="install"
PKG_UPDATE_ARG="update"
@ -522,57 +522,15 @@ function set_bin_path() {
function cmd_set_datastore() {
local IMAGEID=$1
local DATASTORE_TYPE=$2
# rd_manage datastore_update <datastore_name> <default_version>
rd_manage datastore_update "$DATASTORE_TYPE" ""
PACKAGES=${PACKAGES:-""}
if [ "$DATASTORE_TYPE" == "mysql" ]; then
VERSION="5.7"
elif [ "$DATASTORE_TYPE" == "percona" ]; then
PACKAGES=${PACKAGES:-"percona-server-server-5.6"}
VERSION="5.6"
elif [ "$DATASTORE_TYPE" == "pxc" ]; then
PACKAGES=${PACKAGES:-"percona-xtradb-cluster-server-5.6"}
VERSION="5.6"
elif [ "$DATASTORE_TYPE" == "mariadb" ]; then
VERSION="10.4"
elif [ "$DATASTORE_TYPE" == "mongodb" ]; then
PACKAGES=${PACKAGES:-"mongodb-org"}
VERSION="3.2"
elif [ "$DATASTORE_TYPE" == "redis" ]; then
PACKAGES=${PACKAGES:-""}
VERSION="3.2.6"
elif [ "$DATASTORE_TYPE" == "cassandra" ]; then
PACKAGES=${PACKAGES:-"cassandra"}
VERSION="2.1.0"
elif [ "$DATASTORE_TYPE" == "couchbase" ]; then
PACKAGES=${PACKAGES:-"couchbase-server"}
VERSION="2.2.0"
elif [ "$DATASTORE_TYPE" == "postgresql" ]; then
VERSION="9.6"
elif [ "$DATASTORE_TYPE" == "couchdb" ]; then
PACKAGES=${PACKAGES:-"couchdb"}
VERSION="1.6.1"
elif [ "$DATASTORE_TYPE" == "vertica" ]; then
PACKAGES=${PACKAGES:-"vertica"}
VERSION="9.0.1"
elif [ "$DATASTORE_TYPE" == "db2" ]; then
PACKAGES=${PACKAGES:-""}
VERSION="11.1"
else
echo "Unrecognized datastore type. ($DATASTORE_TYPE)"
exit 1
fi
rd_manage datastore_update "$datastore" ""
# trove-manage datastore_version_update <datastore_name> <version_name> <datastore_manager> <image_id> <packages> <active>
rd_manage datastore_version_update "$DATASTORE_TYPE" "$VERSION" "$DATASTORE_TYPE" $IMAGEID "$PACKAGES" 1
rd_manage datastore_update "$DATASTORE_TYPE" "$VERSION"
rd_manage datastore_version_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "${DATASTORE_TYPE}" $IMAGEID "" 1
rd_manage datastore_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}"
if [ -f "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json ]; then
if [[ -f "$PATH_TROVE"/trove/templates/${DATASTORE_TYPE}/validation-rules.json ]]; then
# add the configuration parameters to the database for the kick-start datastore
rd_manage db_load_datastore_config_parameters "$DATASTORE_TYPE" "$VERSION" "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json
rd_manage db_load_datastore_config_parameters "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "$PATH_TROVE"/trove/templates/${DATASTORE_TYPE}/validation-rules.json
fi
}
@ -627,8 +585,8 @@ function install_test_packages() {
DATASTORE_TYPE=$1
sudo -H $HTTP_PROXY pip install openstack.nose_plugin proboscis pexpect
if [ "$DATASTORE_TYPE" = "couchbase" ]; then
if [ "$DISTRO" == "ubuntu" ]; then
if [[ "$DATASTORE_TYPE" = "couchbase" ]]; then
if [[ "$DISTRO" == "ubuntu" ]]; then
# Install Couchbase SDK for scenario tests.
sudo -H $HTTP_PROXY curl http://packages.couchbase.com/ubuntu/couchbase.key | sudo apt-key add -
echo "deb http://packages.couchbase.com/ubuntu trusty trusty/main" | sudo tee /etc/apt/sources.list.d/couchbase-csdk.list
@ -649,12 +607,12 @@ function mod_confs() {
TROVE_REPORT_DIR=${TROVE_REPORT_DIR:=$TROVESTACK_SCRIPTS/../report/}
EXTRA_CONF=$TROVESTACK_SCRIPTS/conf/test.extra.conf
if [ -e $EXTRA_CONF ]; then
if [[ -e $EXTRA_CONF ]]; then
cat $EXTRA_CONF >> $TEST_CONF
fi
# Append datastore specific configuration file
DATASTORE_CONF=$TROVESTACK_SCRIPTS/conf/$DATASTORE_TYPE.conf
if [ ! -f $DATASTORE_CONF ]; then
if [[ ! -f $DATASTORE_CONF ]]; then
exclaim "Datastore configuration file ${DATASTORE_CONF} not found"
exit 1
fi
@ -695,14 +653,14 @@ function mod_confs() {
sed -i "/%shared_network_subnet%/d" $TEST_CONF
fi
if [ "$DATASTORE_TYPE" = "vertica" ]; then
if [[ "$DATASTORE_TYPE" = "vertica" ]]; then
# Vertica needs more time than mysql for its boot/start/stop operations.
setup_cluster_configs cluster_member_count 3
elif [ "$DATASTORE_TYPE" = "pxc" ]; then
elif [[ "$DATASTORE_TYPE" = "pxc" ]]; then
setup_cluster_configs min_cluster_member_count 2
elif [ "$DATASTORE_TYPE" = "cassandra" ]; then
elif [[ "$DATASTORE_TYPE" = "cassandra" ]]; then
setup_cluster_configs cluster_member_count 2
elif [ "$DATASTORE_TYPE" = "mongodb" ]; then
elif [[ "$DATASTORE_TYPE" = "mongodb" ]]; then
setup_cluster_configs cluster_member_count 2
# Decrease the number of required config servers per cluster to save resources.
iniset $TROVE_CONF $DATASTORE_TYPE num_config_servers_per_cluster 1
@ -747,7 +705,7 @@ function cmd_test_init() {
local DATASTORE_TYPE=$1
local DATASTORE_VERSION=$2
if [ -z "${DATASTORE_TYPE}" ]; then
if [[ -z "${DATASTORE_TYPE}" ]]; then
exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}"
exit 1
fi
@ -768,73 +726,60 @@ function cmd_test_init() {
pip3 install -U git+https://opendev.org/openstack/python-troveclient@master#egg=python-troveclient
}
# Build trove guest image
function cmd_build_image() {
exclaim "Params for cmd_build_image function: $@"
local IMAGE_DATASTORE_TYPE=${1:-'mysql'}
local IMAGE_GUEST_OS=${2:-'ubuntu'}
local IMAGE_GUEST_RELEASE=${3:-'xenial'}
local DEV_MODE=${4:-'true'}
local guest_username=${5:-'ubuntu'}
local output=$6
local image_guest_os=${1:-'ubuntu'}
local image_guest_release=${2:-'bionic'}
local dev_mode=${3:-'true'}
local guest_username=${4:-'ubuntu'}
local output=$5
if [[ -z "$output" ]]; then
image_name="trove-datastore-${IMAGE_GUEST_OS}-${IMAGE_GUEST_RELEASE}-${IMAGE_DATASTORE_TYPE}"
image_folder=$HOME/images
output="${image_folder}/${image_name}"
image_name="trove-guest-${image_guest_os}-${image_guest_release}"
if [[ ${dev_mode} == "true" ]]; then
image_name="${image_name}-dev"
fi
image_folder=$HOME/images
output="${image_folder}/${image_name}.qcow2"
fi
# Always rebuild the image.
sudo rm -f $output
sudo mkdir -p $(dirname $output); sudo chmod 777 -R $(dirname $output)
sudo rm -rf ${output}
sudo mkdir -p $(dirname ${output}); sudo chmod 777 -R $(dirname ${output})
echo "Ensuring we have all packages needed to build image."
sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS update
sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu git kpartx debootstrap
sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu git kpartx debootstrap squashfs-tools
sudo -H $HTTP_PROXY pip install diskimage-builder
exclaim "Use diskimage-builder to actually build the Trove Guest Agent Image."
build_guest_image $IMAGE_DATASTORE_TYPE $IMAGE_GUEST_OS $IMAGE_GUEST_RELEASE $DEV_MODE ${guest_username} $output
build_guest_image ${image_guest_os} ${image_guest_release} ${dev_mode} ${guest_username} ${output}
}
# Build guest image and upload to Glance, register the datastore and configuration parameters.
# We could skip the image build and upload by:
# 1. MYSQL_IMAGE_ID is passed, or
# 2. There is an image in Glance contains the datastore name
function cmd_build_and_upload_image() {
local datastore_type=$1
local guest_os=${2:-"ubuntu"}
local guest_release=${3:-"xenial"}
local dev_mode=${4:-"true"}
local guest_username=${5:-"ubuntu"}
local output_dir=${6:-"$HOME/images"}
local guest_os=${1:-"ubuntu"}
local guest_release=${2:-"bionic"}
local dev_mode=${3:-"true"}
local guest_username=${4:-"ubuntu"}
local output_dir=${5:-"$HOME/images"}
if [ -z "${datastore_type}" ]; then
exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}"
exit 1
fi
image_var="${datastore_type^^}_IMAGE_ID"
glance_imageid=`eval echo '$'"$image_var"`
if [[ -z $glance_imageid ]]; then
# Find the first image id with the name contains datastore_type.
glance_imageid=$(openstack $CLOUD_ADMIN_ARG image list | grep "$datastore_type" | awk 'NR==1 {print}' | awk '{print $2}')
if [[ -z $glance_imageid ]]; then
name=trove-guest-${guest_os}-${guest_release}
glance_imageid=$(openstack ${CLOUD_ADMIN_ARG} image list --name $name -f value -c ID)
if [[ -z ${glance_imageid} ]]; then
mkdir -p ${output_dir}
name=trove-datastore-${guest_os}-${guest_release}-${datastore_type}
output=${output_dir}/$name.qcow2
cmd_build_image ${datastore_type} ${guest_os} ${guest_release} ${dev_mode} ${guest_username} $output
output=${output_dir}/${name}
cmd_build_image ${guest_os} ${guest_release} ${dev_mode} ${guest_username} ${output}
glance_imageid=$(openstack ${CLOUD_ADMIN_ARG} image create $name --public --disk-format qcow2 --container-format bare --file $output --property hw_rng_model='virtio' -c id -f value)
glance_imageid=$(openstack ${CLOUD_ADMIN_ARG} image create ${name} --public --disk-format qcow2 --container-format bare --file ${output} --property hw_rng_model='virtio' --tag trove -c id -f value)
[[ -z "$glance_imageid" ]] && echo "Glance upload failed!" && exit 1
fi
fi
exclaim "Using Glance image ID: $glance_imageid"
exclaim "Updating Datastores"
cmd_set_datastore "${glance_imageid}" "${datastore_type}"
cmd_set_datastore "${glance_imageid}"
}
@ -991,11 +936,11 @@ function cmd_stop() {
function cmd_int_tests() {
exclaim "Running Trove Integration Tests..."
if [ ! $USAGE_ENDPOINT ]; then
if [[ ! $USAGE_ENDPOINT ]]; then
export USAGE_ENDPOINT=trove.tests.util.usage.FakeVerifier
fi
cd $TROVESTACK_SCRIPTS
if [ $# -lt 1 ]; then
if [[ $# -lt 1 ]]; then
args="--group=mysql"
else
args="$@"
@ -1203,7 +1148,7 @@ function cmd_kick_start() {
local DATASTORE_TYPE=$1
local DATASTORE_VERSION=$2
if [ -z "${DATASTORE_TYPE}" ]; then
if [[ -z "${DATASTORE_TYPE}" ]]; then
exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}"
exit 1
fi
@ -1220,10 +1165,13 @@ function cmd_kick_start() {
function cmd_gate_tests() {
local DATASTORE_TYPE=${1:-'mysql'}
local TEST_GROUP=${2:-${DATASTORE_TYPE}}
local DATASTORE_VERSION=${3:-'5.7'}
local DATASTORE_VERSION=${3:-'5.7.29'}
local HOST_SCP_USERNAME=${4:-$(whoami)}
local GUEST_USERNAME=${5:-'ubuntu'}
export DATASTORE_TYPE=${DATASTORE_TYPE}
export DATASTORE_VERSION=${DATASTORE_VERSION}
exclaim "Running cmd_gate_tests ..."
export REPORT_DIRECTORY=${REPORT_DIRECTORY:=$HOME/gate-tests-report/}
@ -1238,7 +1186,7 @@ function cmd_gate_tests() {
cd $TROVESTACK_SCRIPTS
# Build and upload guest image, register datastore version.
cmd_build_and_upload_image ${DATASTORE_TYPE}
cmd_build_and_upload_image
cmd_kick_start "${DATASTORE_TYPE}" "${DATASTORE_VERSION}"

View File

@ -25,6 +25,7 @@ deprecation==2.0
diskimage-builder==1.1.2
doc8==0.6.0
docutils==0.14
docker==4.2.0
dogpile.cache==0.6.5
dulwich==0.19.0
enum34===1.0.4

View File

@ -7,12 +7,11 @@
- name: Build Trove guest image
shell: >-
./trovestack build-image \
{{ datastore_type }} \
{{ guest_os }} \
{{ guest_os_release }} \
{{ dev_mode }} \
{{ guest_username }} \
{{ ansible_user_dir }}/images/trove-{{ branch }}-{{ datastore_type }}-{{ guest_os }}-{{ guest_os_release }}{{ image_suffix }}
{{ ansible_user_dir }}/images/trove-{{ branch }}-guest-{{ guest_os }}-{{ guest_os_release }}{{ image_suffix }}.qcow2
args:
chdir: "{{ ansible_user_dir }}/src/opendev.org/openstack/trove/integration/scripts"
tags:

View File

@ -48,3 +48,4 @@ xmltodict>=0.10.1 # MIT
cryptography>=2.1.4 # BSD/Apache-2.0
oslo.policy>=1.30.0 # Apache-2.0
diskimage-builder!=1.6.0,!=1.7.0,!=1.7.1,>=1.1.2 # Apache-2.0
docker>=4.2.0 # Apache-2.0

View File

@ -1,5 +1,5 @@
devstack_base_dir: /opt/stack
trove_test_datastore: 'mysql'
trove_test_group: 'mysql'
trove_test_datastore_version: '5.7'
trove_test_datastore_version: '5.7.29'
trove_resize_time_out: ''

View File

@ -1497,6 +1497,12 @@
"Instance of 'FreshInstance' has no 'get_replication_master_snapshot' member",
"Manager._create_replication_slave"
],
[
"trove/taskmanager/manager.py",
"E1136",
"Value 'snapshot' is unsubscriptable",
"Manager._create_replication_slave"
],
[
"trove/taskmanager/manager.py",
"E1101",

View File

@ -53,9 +53,7 @@ ignore-path = .venv,.tox,.git,dist,doc,*egg-info,tools,etc,build,*.po,*.pot,inte
[flake8]
show-source = True
# H301 is ignored on purpose.
# The rest of the ignores are TODOs.
ignore = E402,E731,F601,F821,H301,H404,H405,H501,W503,W504,W605
ignore = E125,E129,E402,E731,F601,F821,H301,H306,H404,H405,H501,W503,W504,W605
enable-extensions = H203,H106
builtins = _
# add *.yaml for playbooks/trove-devstack-base.yaml, as it will be matched by
@ -68,7 +66,7 @@ import_exceptions = trove.common.i18n
[flake8:local-plugins]
extension =
T103= checks:check_raised_localized_exceptions
# T103= checks:check_raised_localized_exceptions
T104 = checks:check_no_basestring
T105 = checks:no_translate_logs
N335 = checks:assert_raises_regexp

View File

@ -268,7 +268,7 @@ class Backup(object):
try:
cls.delete(context, child.id)
except exception.NotFound:
LOG.exception("Backup %s cannot be found.", backup_id)
LOG.warning("Backup %s cannot be found.", backup_id)
def _delete_resources():
backup = cls.get_by_id(context, backup_id)

View File

@ -12,7 +12,6 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
from oslo_config import cfg as openstack_cfg
@ -23,6 +22,7 @@ from trove.common import cfg
from trove.common import debug_utils
from trove.common.i18n import _
from trove.guestagent import api as guest_api
from trove.guestagent.common import operating_system
CONF = cfg.CONF
# The guest_id opt definition must match the one in common/cfg.py
@ -31,9 +31,18 @@ CONF.register_opts([openstack_cfg.StrOpt('guest_id', default=None,
openstack_cfg.StrOpt('instance_rpc_encr_key',
help=('Key (OpenSSL aes_cbc) for '
'instance RPC encryption.'))])
LOG = logging.getLogger(__name__)
def main():
log_levels = [
'docker=WARN',
]
default_log_levels = logging.get_default_log_levels()
default_log_levels.extend(log_levels)
logging.set_defaults(default_log_levels=default_log_levels)
logging.register_options(CONF)
cfg.parse_args(sys.argv)
logging.setup(CONF, None)
debug_utils.setup()
@ -50,6 +59,11 @@ def main():
"was not injected into the guest or not read by guestagent"))
raise RuntimeError(msg)
# Create user and group for running docker container.
LOG.info('Creating user and group for database service')
uid = cfg.get_configuration_property('database_service_uid')
operating_system.create_user('database', uid)
# rpc module must be loaded after decision about thread monkeypatching
# because if thread module is not monkeypatched we can't use eventlet
# executor from oslo_messaging library.

View File

@ -175,10 +175,10 @@ common_opts = [
help="Maximum time (in seconds) to wait for Guest Agent "
"'quick' requests (such as retrieving a list of "
"users or databases)."),
cfg.IntOpt('agent_call_high_timeout', default=60 * 5,
cfg.IntOpt('agent_call_high_timeout', default=60 * 3,
help="Maximum time (in seconds) to wait for Guest Agent 'slow' "
"requests (such as restarting the database)."),
cfg.IntOpt('agent_replication_snapshot_timeout', default=36000,
cfg.IntOpt('agent_replication_snapshot_timeout', default=60 * 30,
help='Maximum time (in seconds) to wait for taking a Guest '
'Agent replication snapshot.'),
cfg.IntOpt('command_process_timeout', default=30,
@ -186,8 +186,9 @@ common_opts = [
'commands to complete.'),
# The guest_id opt definition must match the one in cmd/guest.py
cfg.StrOpt('guest_id', default=None, help="ID of the Guest Instance."),
cfg.IntOpt('state_change_wait_time', default=60 * 10,
help='Maximum time (in seconds) to wait for a state change.'),
cfg.IntOpt('state_change_wait_time', default=180,
help='Maximum time (in seconds) to wait for database state '
'change.'),
cfg.IntOpt('state_change_poll_time', default=3,
help='Interval between state change poll requests (seconds).'),
cfg.IntOpt('agent_heartbeat_time', default=10,
@ -293,9 +294,11 @@ common_opts = [
help='The region this service is located.'),
cfg.StrOpt('backup_runner',
default='trove.guestagent.backup.backup_types.InnoBackupEx',
help='Runner to use for backups.'),
help='Runner to use for backups.',
deprecated_for_removal=True),
cfg.DictOpt('backup_runner_options', default={},
help='Additional options to be passed to the backup runner.'),
help='Additional options to be passed to the backup runner.',
deprecated_for_removal=True),
cfg.BoolOpt('verify_swift_checksum_on_restore', default=True,
help='Enable verification of Swift checksum before starting '
'restore. Makes sure the checksum of original backup matches '
@ -304,11 +307,12 @@ common_opts = [
help='Require the replica volume size to be greater than '
'or equal to the size of the master volume '
'during replica creation.'),
cfg.StrOpt('storage_strategy', default='SwiftStorage',
cfg.StrOpt('storage_strategy', default='swift',
help="Default strategy to store backups."),
cfg.StrOpt('storage_namespace',
default='trove.common.strategies.storage.swift',
help='Namespace to load the default storage strategy from.'),
help='Namespace to load the default storage strategy from.',
deprecated_for_removal=True),
cfg.StrOpt('backup_swift_container', default='database_backups',
help='Swift container to put backups in.'),
cfg.BoolOpt('backup_use_gzip_compression', default=True,
@ -429,15 +433,12 @@ common_opts = [
cfg.IntOpt('usage_timeout', default=60 * 30,
help='Maximum time (in seconds) to wait for a Guest to become '
'active.'),
cfg.IntOpt('restore_usage_timeout', default=36000,
cfg.IntOpt('restore_usage_timeout', default=60 * 60,
help='Maximum time (in seconds) to wait for a Guest instance '
'restored from a backup to become active.'),
cfg.IntOpt('cluster_usage_timeout', default=36000,
help='Maximum time (in seconds) to wait for a cluster to '
'become active.'),
cfg.IntOpt('timeout_wait_for_service', default=120,
help='Maximum time (in seconds) to wait for a service to '
'become alive.'),
cfg.StrOpt('module_aes_cbc_key', default='module_aes_cbc_key',
help='OpenSSL aes_cbc key for module encryption.'),
cfg.ListOpt('module_types', default=['ping', 'new_relic_license'],
@ -466,6 +467,10 @@ common_opts = [
help='Key (OpenSSL aes_cbc) to encrypt instance keys in DB.'),
cfg.StrOpt('instance_rpc_encr_key',
help='Key (OpenSSL aes_cbc) for instance RPC encryption.'),
cfg.StrOpt('database_service_uid', default='1001',
help='The UID(GID) of database service user.'),
cfg.StrOpt('backup_docker_image', default='openstacktrove/db-backup:1.0.0',
help='The docker image used for backup and restore.'),
]
@ -544,7 +549,7 @@ mysql_opts = [
help='List of UDP ports and/or port ranges to open '
'in the security group (only applicable '
'if trove_security_groups_support is True).'),
cfg.StrOpt('backup_strategy', default='InnoBackupEx',
cfg.StrOpt('backup_strategy', default='innobackupex',
help='Default strategy to perform backups.',
deprecated_name='backup_strategy',
deprecated_group='DEFAULT'),
@ -564,28 +569,10 @@ mysql_opts = [
cfg.IntOpt('usage_timeout', default=400,
help='Maximum time (in seconds) to wait for a Guest to become '
'active.'),
cfg.StrOpt('backup_namespace',
default='trove.guestagent.strategies.backup.mysql_impl',
help='Namespace to load backup strategies from.',
deprecated_name='backup_namespace',
deprecated_group='DEFAULT'),
cfg.StrOpt('restore_namespace',
default='trove.guestagent.strategies.restore.mysql_impl',
help='Namespace to load restore strategies from.',
deprecated_name='restore_namespace',
deprecated_group='DEFAULT'),
cfg.BoolOpt('volume_support', default=True,
help='Whether to provision a Cinder volume for datadir.'),
cfg.StrOpt('device_path', default='/dev/vdb',
help='Device path for volume if volume support is enabled.'),
cfg.DictOpt('backup_incremental_strategy',
default={'InnoBackupEx': 'InnoBackupExIncremental'},
help='Incremental Backup Runner based on the default '
'strategy. For strategies that do not implement an '
'incremental backup, the runner will use the default full '
'backup.',
deprecated_name='backup_incremental_strategy',
deprecated_group='DEFAULT'),
cfg.StrOpt('root_controller',
default='trove.extensions.common.service.DefaultRootController',
help='Root controller implementation for mysql.'),
@ -611,6 +598,10 @@ mysql_opts = [
help='Character length of generated passwords.',
deprecated_name='default_password_length',
deprecated_group='DEFAULT'),
cfg.StrOpt(
'docker_image', default='mysql',
help='Database docker image.'
)
]
# Percona
@ -653,28 +644,10 @@ percona_opts = [
cfg.IntOpt('usage_timeout', default=450,
help='Maximum time (in seconds) to wait for a Guest to become '
'active.'),
cfg.StrOpt('backup_namespace',
default='trove.guestagent.strategies.backup.mysql_impl',
help='Namespace to load backup strategies from.',
deprecated_name='backup_namespace',
deprecated_group='DEFAULT'),
cfg.StrOpt('restore_namespace',
default='trove.guestagent.strategies.restore.mysql_impl',
help='Namespace to load restore strategies from.',
deprecated_name='restore_namespace',
deprecated_group='DEFAULT'),
cfg.BoolOpt('volume_support', default=True,
help='Whether to provision a Cinder volume for datadir.'),
cfg.StrOpt('device_path', default='/dev/vdb',
help='Device path for volume if volume support is enabled.'),
cfg.DictOpt('backup_incremental_strategy',
default={'InnoBackupEx': 'InnoBackupExIncremental'},
help='Incremental Backup Runner based on the default '
'strategy. For strategies that do not implement an '
'incremental backup, the runner will use the default full '
'backup.',
deprecated_name='backup_incremental_strategy',
deprecated_group='DEFAULT'),
cfg.StrOpt('root_controller',
default='trove.extensions.common.service.DefaultRootController',
help='Root controller implementation for percona.'),
@ -739,22 +712,10 @@ pxc_opts = [
cfg.IntOpt('usage_timeout', default=450,
help='Maximum time (in seconds) to wait for a Guest to become '
'active.'),
cfg.StrOpt('backup_namespace',
default='trove.guestagent.strategies.backup.mysql_impl',
help='Namespace to load backup strategies from.'),
cfg.StrOpt('restore_namespace',
default='trove.guestagent.strategies.restore.mysql_impl',
help='Namespace to load restore strategies from.'),
cfg.BoolOpt('volume_support', default=True,
help='Whether to provision a Cinder volume for datadir.'),
cfg.StrOpt('device_path', default='/dev/vdb',
help='Device path for volume if volume support is enabled.'),
cfg.DictOpt('backup_incremental_strategy',
default={'InnoBackupEx': 'InnoBackupExIncremental'},
help='Incremental Backup Runner based on the default '
'strategy. For strategies that do not implement an '
'incremental backup, the runner will use the default full '
'backup.'),
cfg.ListOpt('ignore_users', default=['os_admin', 'root', 'clusterrepuser'],
help='Users to exclude when listing users.'),
cfg.ListOpt('ignore_dbs',
@ -818,12 +779,6 @@ redis_opts = [
help='Default strategy to perform backups.',
deprecated_name='backup_strategy',
deprecated_group='DEFAULT'),
cfg.DictOpt('backup_incremental_strategy', default={},
help='Incremental Backup Runner based on the default '
'strategy. For strategies that do not implement an '
'incremental, the runner will use the default full backup.',
deprecated_name='backup_incremental_strategy',
deprecated_group='DEFAULT'),
cfg.StrOpt('replication_strategy', default='RedisSyncReplication',
help='Default strategy for replication.'),
cfg.StrOpt('replication_namespace',
@ -837,18 +792,6 @@ redis_opts = [
help='Whether to provision a Cinder volume for datadir.'),
cfg.StrOpt('device_path', default='/dev/vdb',
help='Device path for volume if volume support is enabled.'),
cfg.StrOpt('backup_namespace',
default="trove.guestagent.strategies.backup.experimental."
"redis_impl",
help='Namespace to load backup strategies from.',
deprecated_name='backup_namespace',
deprecated_group='DEFAULT'),
cfg.StrOpt('restore_namespace',
default="trove.guestagent.strategies.restore.experimental."
"redis_impl",
help='Namespace to load restore strategies from.',
deprecated_name='restore_namespace',
deprecated_group='DEFAULT'),
cfg.BoolOpt('cluster_support', default=True,
help='Enable clusters to be created and managed.'),
cfg.StrOpt('api_strategy',
@ -893,12 +836,6 @@ cassandra_opts = [
help='List of UDP ports and/or port ranges to open '
'in the security group (only applicable '
'if trove_security_groups_support is True).'),
cfg.DictOpt('backup_incremental_strategy', default={},
help='Incremental strategy based on the default backup '
'strategy. For strategies that do not implement incremental '
'backups, the runner performs full backup instead.',
deprecated_name='backup_incremental_strategy',
deprecated_group='DEFAULT'),
cfg.StrOpt('backup_strategy', default="NodetoolSnapshot",
help='Default strategy to perform backups.',
deprecated_name='backup_strategy',
@ -912,18 +849,6 @@ cassandra_opts = [
help='Whether to provision a Cinder volume for datadir.'),
cfg.StrOpt('device_path', default='/dev/vdb',
help='Device path for volume if volume support is enabled.'),
cfg.StrOpt('backup_namespace',
default="trove.guestagent.strategies.backup.experimental."
"cassandra_impl",
help='Namespace to load backup strategies from.',
deprecated_name='backup_namespace',
deprecated_group='DEFAULT'),
cfg.StrOpt('restore_namespace',
default="trove.guestagent.strategies.restore.experimental."
"cassandra_impl",
help='Namespace to load restore strategies from.',
deprecated_name='restore_namespace',
deprecated_group='DEFAULT'),
cfg.StrOpt('root_controller',
default='trove.extensions.common.service.DefaultRootController',
help='Root controller implementation for Cassandra.'),
@ -1002,12 +927,6 @@ couchbase_opts = [
help='Default strategy to perform backups.',
deprecated_name='backup_strategy',
deprecated_group='DEFAULT'),
cfg.DictOpt('backup_incremental_strategy', default={},
help='Incremental Backup Runner based on the default '
'strategy. For strategies that do not implement an '
'incremental, the runner will use the default full backup.',
deprecated_name='backup_incremental_strategy',
deprecated_group='DEFAULT'),
cfg.StrOpt('replication_strategy', default=None,
help='Default strategy for replication.'),
cfg.StrOpt('mount_point', default='/var/lib/couchbase',
@ -1018,18 +937,6 @@ couchbase_opts = [
'service during instance-create. The generated password for '
'the root user is immediately returned in the response of '
"instance-create as the 'password' field."),
cfg.StrOpt('backup_namespace',
default='trove.guestagent.strategies.backup.experimental.'
'couchbase_impl',
help='Namespace to load backup strategies from.',
deprecated_name='backup_namespace',
deprecated_group='DEFAULT'),
cfg.StrOpt('restore_namespace',
default='trove.guestagent.strategies.restore.experimental.'
'couchbase_impl',
help='Namespace to load restore strategies from.',
deprecated_name='restore_namespace',
deprecated_group='DEFAULT'),
cfg.BoolOpt('volume_support', default=True,
help='Whether to provision a Cinder volume for datadir.'),
cfg.StrOpt('device_path', default='/dev/vdb',
@ -1066,12 +973,6 @@ mongodb_opts = [
help='Default strategy to perform backups.',
deprecated_name='backup_strategy',
deprecated_group='DEFAULT'),
cfg.DictOpt('backup_incremental_strategy', default={},
help='Incremental Backup Runner based on the default '
'strategy. For strategies that do not implement an '
'incremental, the runner will use the default full backup.',
deprecated_name='backup_incremental_strategy',
deprecated_group='DEFAULT'),
cfg.StrOpt('replication_strategy', default=None,
help='Default strategy for replication.'),
cfg.StrOpt('mount_point', default='/var/lib/mongodb',
@ -1109,18 +1010,6 @@ mongodb_opts = [
'mongodb.guestagent.MongoDbGuestAgentStrategy',
help='Class that implements datastore-specific Guest Agent API '
'logic.'),
cfg.StrOpt('backup_namespace',
default='trove.guestagent.strategies.backup.experimental.'
'mongo_impl',
help='Namespace to load backup strategies from.',
deprecated_name='backup_namespace',
deprecated_group='DEFAULT'),
cfg.StrOpt('restore_namespace',
default='trove.guestagent.strategies.restore.experimental.'
'mongo_impl',
help='Namespace to load restore strategies from.',
deprecated_name='restore_namespace',
deprecated_group='DEFAULT'),
cfg.PortOpt('mongodb_port', default=27017,
help='Port for mongod and mongos instances.'),
cfg.PortOpt('configsvr_port', default=27019,
@ -1164,11 +1053,6 @@ postgresql_opts = [
help='The TCP port the server listens on.'),
cfg.StrOpt('backup_strategy', default='PgBaseBackup',
help='Default strategy to perform backups.'),
cfg.DictOpt('backup_incremental_strategy',
default={'PgBaseBackup': 'PgBaseBackupIncremental'},
help='Incremental Backup Runner based on the default '
'strategy. For strategies that do not implement an '
'incremental, the runner will use the default full backup.'),
cfg.StrOpt('replication_strategy',
default='PostgresqlReplicationStreaming',
help='Default strategy for replication.'),
@ -1188,14 +1072,6 @@ postgresql_opts = [
'service during instance-create. The generated password for '
'the root user is immediately returned in the response of '
"instance-create as the 'password' field."),
cfg.StrOpt('backup_namespace',
default='trove.guestagent.strategies.backup.experimental.'
'postgresql_impl',
help='Namespace to load backup strategies from.'),
cfg.StrOpt('restore_namespace',
default='trove.guestagent.strategies.restore.experimental.'
'postgresql_impl',
help='Namespace to load restore strategies from.'),
cfg.BoolOpt('volume_support', default=True,
help='Whether to provision a Cinder volume for datadir.'),
cfg.StrOpt('device_path', default='/dev/vdb'),
@ -1248,16 +1124,6 @@ couchdb_opts = [
help='Default strategy to perform backups.'),
cfg.StrOpt('replication_strategy', default=None,
help='Default strategy for replication.'),
cfg.StrOpt('backup_namespace', default='trove.guestagent.strategies'
'.backup.experimental.couchdb_impl',
help='Namespace to load backup strategies from.'),
cfg.StrOpt('restore_namespace', default='trove.guestagent.strategies'
'.restore.experimental.couchdb_impl',
help='Namespace to load restore strategies from.'),
cfg.DictOpt('backup_incremental_strategy', default={},
help='Incremental Backup Runner based on the default '
'strategy. For strategies that do not implement an '
'incremental, the runner will use the default full backup.'),
cfg.BoolOpt('root_on_create', default=False,
help='Enable the automatic creation of the root user for the '
'service during instance-create. The generated password for '
@ -1303,10 +1169,6 @@ vertica_opts = [
'if trove_security_groups_support is True).'),
cfg.StrOpt('backup_strategy', default=None,
help='Default strategy to perform backups.'),
cfg.DictOpt('backup_incremental_strategy', default={},
help='Incremental Backup Runner based on the default '
'strategy. For strategies that do not implement an '
'incremental, the runner will use the default full backup.'),
cfg.StrOpt('replication_strategy', default=None,
help='Default strategy for replication.'),
cfg.StrOpt('mount_point', default='/var/lib/vertica',
@ -1317,9 +1179,11 @@ vertica_opts = [
cfg.StrOpt('device_path', default='/dev/vdb',
help='Device path for volume if volume support is enabled.'),
cfg.StrOpt('backup_namespace', default=None,
help='Namespace to load backup strategies from.'),
help='Namespace to load backup strategies from.',
deprecated_for_removal=True),
cfg.StrOpt('restore_namespace', default=None,
help='Namespace to load restore strategies from.'),
help='Namespace to load restore strategies from.',
deprecated_for_removal=True),
cfg.IntOpt('readahead_size', default=2048,
help='Size(MB) to be set as readahead_size for data volume'),
cfg.BoolOpt('cluster_support', default=True,
@ -1387,22 +1251,6 @@ db2_opts = [
'service during instance-create. The generated password for '
'the root user is immediately returned in the response of '
"instance-create as the 'password' field."),
cfg.StrOpt('backup_namespace',
default='trove.guestagent.strategies.backup.experimental.'
'db2_impl',
help='Namespace to load backup strategies from.',
deprecated_name='backup_namespace',
deprecated_group='DEFAULT'),
cfg.StrOpt('restore_namespace',
default='trove.guestagent.strategies.restore.experimental.'
'db2_impl',
help='Namespace to load restore strategies from.',
deprecated_name='restore_namespace',
deprecated_group='DEFAULT'),
cfg.DictOpt('backup_incremental_strategy', default={},
help='Incremental Backup Runner based on the default '
'strategy. For strategies that do not implement an '
'incremental, the runner will use the default full backup.'),
cfg.ListOpt('ignore_users', default=['PUBLIC', 'DB2INST1']),
cfg.StrOpt('root_controller',
default='trove.extensions.common.service.DefaultRootController',
@ -1432,21 +1280,14 @@ mariadb_opts = [
help='List of UDP ports and/or port ranges to open '
'in the security group (only applicable '
'if trove_security_groups_support is True).'),
cfg.StrOpt('backup_namespace',
default='trove.guestagent.strategies.backup.experimental'
'.mariadb_impl',
help='Namespace to load backup strategies from.',
deprecated_name='backup_namespace',
deprecated_group='DEFAULT'),
cfg.StrOpt('backup_strategy', default='MariaBackup',
cfg.StrOpt('backup_strategy', default='mariabackup',
help='Default strategy to perform backups.',
deprecated_name='backup_strategy',
deprecated_group='DEFAULT'),
cfg.StrOpt('replication_strategy', default='MariaDBGTIDReplication',
help='Default strategy for replication.'),
cfg.StrOpt('replication_namespace',
default='trove.guestagent.strategies.replication.experimental'
'.mariadb_gtid',
default='trove.guestagent.strategies.replication.mariadb_gtid',
help='Namespace to load replication strategies from.'),
cfg.StrOpt('mount_point', default='/var/lib/mysql',
help="Filesystem path for mounting "
@ -1459,25 +1300,10 @@ mariadb_opts = [
cfg.IntOpt('usage_timeout', default=400,
help='Maximum time (in seconds) to wait for a Guest to become '
'active.'),
cfg.StrOpt('restore_namespace',
default='trove.guestagent.strategies.restore.experimental'
'.mariadb_impl',
help='Namespace to load restore strategies from.',
deprecated_name='restore_namespace',
deprecated_group='DEFAULT'),
cfg.BoolOpt('volume_support', default=True,
help='Whether to provision a Cinder volume for datadir.'),
cfg.StrOpt('device_path', default='/dev/vdb',
help='Device path for volume if volume support is enabled.'),
cfg.DictOpt('backup_incremental_strategy',
default={'MariaBackup':
'MariaBackupIncremental'},
help='Incremental Backup Runner based on the default '
'strategy. For strategies that do not implement an '
'incremental backup, the runner will use the default full '
'backup.',
deprecated_name='backup_incremental_strategy',
deprecated_group='DEFAULT'),
cfg.StrOpt('root_controller',
default='trove.extensions.common.service.DefaultRootController',
help='Root controller implementation for mysql.'),
@ -1521,6 +1347,10 @@ mariadb_opts = [
help='Character length of generated passwords.',
deprecated_name='default_password_length',
deprecated_group='DEFAULT'),
cfg.StrOpt(
'docker_image', default='mariadb',
help='Database docker image.'
)
]
# RPC version groups

View File

@ -307,7 +307,7 @@ class VolumeAttachmentsNotFound(NotFound):
class VolumeCreationFailure(TroveError):
message = _("Failed to create a volume in Nova.")
message = _("Failed to create volume.")
class VolumeSizeNotSpecified(BadRequest):
@ -341,6 +341,16 @@ class ReplicationSlaveAttachError(TroveError):
message = _("Exception encountered attaching slave to new replica source.")
class SlaveOperationNotSupported(TroveError):
message = _("The '%(operation)s' operation is not supported for slaves in "
"replication.")
class UnableToDetermineLastMasterGTID(TroveError):
message = _("Unable to determine last GTID executed on master "
"(from file %(binlog_file)s).")
class TaskManagerError(TroveError):
message = _("An error occurred communicating with the task manager: "
@ -688,9 +698,3 @@ class LogAccessForbidden(Forbidden):
class LogsNotAvailable(Forbidden):
message = _("Log actions are not supported.")
class SlaveOperationNotSupported(TroveError):
message = _("The '%(operation)s' operation is not supported for slaves in "
"replication.")

View File

@ -1,141 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
-*- rnc -*-
RELAX NG Compact Syntax Grammar for the
Atom Format Specification Version 11
-->
<grammar xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:s="http://www.ascc.net/xml/schematron" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
<start>
<choice>
<ref name="atomLink"/>
</choice>
</start>
<!-- Common attributes -->
<define name="atomCommonAttributes">
<optional>
<attribute name="xml:base">
<ref name="atomUri"/>
</attribute>
</optional>
<optional>
<attribute name="xml:lang">
<ref name="atomLanguageTag"/>
</attribute>
</optional>
<zeroOrMore>
<ref name="undefinedAttribute"/>
</zeroOrMore>
</define>
<!-- atom:link -->
<define name="atomLink">
<element name="atom:link">
<ref name="atomCommonAttributes"/>
<attribute name="href">
<ref name="atomUri"/>
</attribute>
<optional>
<attribute name="rel">
<choice>
<ref name="atomNCName"/>
<ref name="atomUri"/>
</choice>
</attribute>
</optional>
<optional>
<attribute name="type">
<ref name="atomMediaType"/>
</attribute>
</optional>
<optional>
<attribute name="hreflang">
<ref name="atomLanguageTag"/>
</attribute>
</optional>
<optional>
<attribute name="title"/>
</optional>
<optional>
<attribute name="length"/>
</optional>
<ref name="undefinedContent"/>
</element>
</define>
<!-- Low-level simple types -->
<define name="atomNCName">
<data type="string">
<param name="minLength">1</param>
<param name="pattern">[^:]*</param>
</data>
</define>
<!-- Whatever a media type is, it contains at least one slash -->
<define name="atomMediaType">
<data type="string">
<param name="pattern">.+/.+</param>
</data>
</define>
<!-- As defined in RFC 3066 -->
<define name="atomLanguageTag">
<data type="string">
<param name="pattern">[A-Za-z]{1,8}(-[A-Za-z0-9]{1,8})*</param>
</data>
</define>
<!--
Unconstrained; it's not entirely clear how IRI fit into
xsd:anyURI so let's not try to constrain it here
-->
<define name="atomUri">
<text/>
</define>
<!-- Other Extensibility -->
<define name="undefinedAttribute">
<attribute>
<anyName>
<except>
<name>xml:base</name>
<name>xml:lang</name>
<nsName ns=""/>
</except>
</anyName>
</attribute>
</define>
<define name="undefinedContent">
<zeroOrMore>
<choice>
<text/>
<ref name="anyForeignElement"/>
</choice>
</zeroOrMore>
</define>
<define name="anyElement">
<element>
<anyName/>
<zeroOrMore>
<choice>
<attribute>
<anyName/>
</attribute>
<text/>
<ref name="anyElement"/>
</choice>
</zeroOrMore>
</element>
</define>
<define name="anyForeignElement">
<element>
<anyName>
<except>
<nsName ns="http://www.w3.org/2005/Atom"/>
</except>
</anyName>
<zeroOrMore>
<choice>
<attribute>
<anyName/>
</attribute>
<text/>
<ref name="anyElement"/>
</choice>
</zeroOrMore>
</element>
</define>
</grammar>

View File

@ -1,597 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
-*- rnc -*-
RELAX NG Compact Syntax Grammar for the
Atom Format Specification Version 11
-->
<grammar xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:s="http://www.ascc.net/xml/schematron" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
<start>
<choice>
<ref name="atomFeed"/>
<ref name="atomEntry"/>
</choice>
</start>
<!-- Common attributes -->
<define name="atomCommonAttributes">
<optional>
<attribute name="xml:base">
<ref name="atomUri"/>
</attribute>
</optional>
<optional>
<attribute name="xml:lang">
<ref name="atomLanguageTag"/>
</attribute>
</optional>
<zeroOrMore>
<ref name="undefinedAttribute"/>
</zeroOrMore>
</define>
<!-- Text Constructs -->
<define name="atomPlainTextConstruct">
<ref name="atomCommonAttributes"/>
<optional>
<attribute name="type">
<choice>
<value>text</value>
<value>html</value>
</choice>
</attribute>
</optional>
<text/>
</define>
<define name="atomXHTMLTextConstruct">
<ref name="atomCommonAttributes"/>
<attribute name="type">
<value>xhtml</value>
</attribute>
<ref name="xhtmlDiv"/>
</define>
<define name="atomTextConstruct">
<choice>
<ref name="atomPlainTextConstruct"/>
<ref name="atomXHTMLTextConstruct"/>
</choice>
</define>
<!-- Person Construct -->
<define name="atomPersonConstruct">
<ref name="atomCommonAttributes"/>
<interleave>
<element name="atom:name">
<text/>
</element>
<optional>
<element name="atom:uri">
<ref name="atomUri"/>
</element>
</optional>
<optional>
<element name="atom:email">
<ref name="atomEmailAddress"/>
</element>
</optional>
<zeroOrMore>
<ref name="extensionElement"/>
</zeroOrMore>
</interleave>
</define>
<!-- Date Construct -->
<define name="atomDateConstruct">
<ref name="atomCommonAttributes"/>
<data type="dateTime"/>
</define>
<!-- atom:feed -->
<define name="atomFeed">
<element name="atom:feed">
<s:rule context="atom:feed">
<s:assert test="atom:author or not(atom:entry[not(atom:author)])">An atom:feed must have an atom:author unless all of its atom:entry children have an atom:author.</s:assert>
</s:rule>
<ref name="atomCommonAttributes"/>
<interleave>
<zeroOrMore>
<ref name="atomAuthor"/>
</zeroOrMore>
<zeroOrMore>
<ref name="atomCategory"/>
</zeroOrMore>
<zeroOrMore>
<ref name="atomContributor"/>
</zeroOrMore>
<optional>
<ref name="atomGenerator"/>
</optional>
<optional>
<ref name="atomIcon"/>
</optional>
<ref name="atomId"/>
<zeroOrMore>
<ref name="atomLink"/>
</zeroOrMore>
<optional>
<ref name="atomLogo"/>
</optional>
<optional>
<ref name="atomRights"/>
</optional>
<optional>
<ref name="atomSubtitle"/>
</optional>
<ref name="atomTitle"/>
<ref name="atomUpdated"/>
<zeroOrMore>
<ref name="extensionElement"/>
</zeroOrMore>
</interleave>
<zeroOrMore>
<ref name="atomEntry"/>
</zeroOrMore>
</element>
</define>
<!-- atom:entry -->
<define name="atomEntry">
<element name="atom:entry">
<s:rule context="atom:entry">
<s:assert test="atom:link[@rel='alternate'] or atom:link[not(@rel)] or atom:content">An atom:entry must have at least one atom:link element with a rel attribute of 'alternate' or an atom:content.</s:assert>
</s:rule>
<s:rule context="atom:entry">
<s:assert test="atom:author or ../atom:author or atom:source/atom:author">An atom:entry must have an atom:author if its feed does not.</s:assert>
</s:rule>
<ref name="atomCommonAttributes"/>
<interleave>
<zeroOrMore>
<ref name="atomAuthor"/>
</zeroOrMore>
<zeroOrMore>
<ref name="atomCategory"/>
</zeroOrMore>
<optional>
<ref name="atomContent"/>
</optional>
<zeroOrMore>
<ref name="atomContributor"/>
</zeroOrMore>
<ref name="atomId"/>
<zeroOrMore>
<ref name="atomLink"/>
</zeroOrMore>
<optional>
<ref name="atomPublished"/>
</optional>
<optional>
<ref name="atomRights"/>
</optional>
<optional>
<ref name="atomSource"/>
</optional>
<optional>
<ref name="atomSummary"/>
</optional>
<ref name="atomTitle"/>
<ref name="atomUpdated"/>
<zeroOrMore>
<ref name="extensionElement"/>
</zeroOrMore>
</interleave>
</element>
</define>
<!-- atom:content -->
<define name="atomInlineTextContent">
<element name="atom:content">
<ref name="atomCommonAttributes"/>
<optional>
<attribute name="type">
<choice>
<value>text</value>
<value>html</value>
</choice>
</attribute>
</optional>
<zeroOrMore>
<text/>
</zeroOrMore>
</element>
</define>
<define name="atomInlineXHTMLContent">
<element name="atom:content">
<ref name="atomCommonAttributes"/>
<attribute name="type">
<value>xhtml</value>
</attribute>
<ref name="xhtmlDiv"/>
</element>
</define>
<define name="atomInlineOtherContent">
<element name="atom:content">
<ref name="atomCommonAttributes"/>
<optional>
<attribute name="type">
<ref name="atomMediaType"/>
</attribute>
</optional>
<zeroOrMore>
<choice>
<text/>
<ref name="anyElement"/>
</choice>
</zeroOrMore>
</element>
</define>
<define name="atomOutOfLineContent">
<element name="atom:content">
<ref name="atomCommonAttributes"/>
<optional>
<attribute name="type">
<ref name="atomMediaType"/>
</attribute>
</optional>
<attribute name="src">
<ref name="atomUri"/>
</attribute>
<empty/>
</element>
</define>
<define name="atomContent">
<choice>
<ref name="atomInlineTextContent"/>
<ref name="atomInlineXHTMLContent"/>
<ref name="atomInlineOtherContent"/>
<ref name="atomOutOfLineContent"/>
</choice>
</define>
<!-- atom:author -->
<define name="atomAuthor">
<element name="atom:author">
<ref name="atomPersonConstruct"/>
</element>
</define>
<!-- atom:category -->
<define name="atomCategory">
<element name="atom:category">
<ref name="atomCommonAttributes"/>
<attribute name="term"/>
<optional>
<attribute name="scheme">
<ref name="atomUri"/>
</attribute>
</optional>
<optional>
<attribute name="label"/>
</optional>
<ref name="undefinedContent"/>
</element>
</define>
<!-- atom:contributor -->
<define name="atomContributor">
<element name="atom:contributor">
<ref name="atomPersonConstruct"/>
</element>
</define>
<!-- atom:generator -->
<define name="atomGenerator">
<element name="atom:generator">
<ref name="atomCommonAttributes"/>
<optional>
<attribute name="uri">
<ref name="atomUri"/>
</attribute>
</optional>
<optional>
<attribute name="version"/>
</optional>
<text/>
</element>
</define>
<!-- atom:icon -->
<define name="atomIcon">
<element name="atom:icon">
<ref name="atomCommonAttributes"/>
<ref name="atomUri"/>
</element>
</define>
<!-- atom:id -->
<define name="atomId">
<element name="atom:id">
<ref name="atomCommonAttributes"/>
<ref name="atomUri"/>
</element>
</define>
<!-- atom:logo -->
<define name="atomLogo">
<element name="atom:logo">
<ref name="atomCommonAttributes"/>
<ref name="atomUri"/>
</element>
</define>
<!-- atom:link -->
<define name="atomLink">
<element name="atom:link">
<ref name="atomCommonAttributes"/>
<attribute name="href">
<ref name="atomUri"/>
</attribute>
<optional>
<attribute name="rel">
<choice>
<ref name="atomNCName"/>
<ref name="atomUri"/>
</choice>
</attribute>
</optional>
<optional>
<attribute name="type">
<ref name="atomMediaType"/>
</attribute>
</optional>
<optional>
<attribute name="hreflang">
<ref name="atomLanguageTag"/>
</attribute>
</optional>
<optional>
<attribute name="title"/>
</optional>
<optional>
<attribute name="length"/>
</optional>
<ref name="undefinedContent"/>
</element>
</define>
<!-- atom:published -->
<define name="atomPublished">
<element name="atom:published">
<ref name="atomDateConstruct"/>
</element>
</define>
<!-- atom:rights -->
<define name="atomRights">
<element name="atom:rights">
<ref name="atomTextConstruct"/>
</element>
</define>
<!-- atom:source -->
<define name="atomSource">
<element name="atom:source">
<ref name="atomCommonAttributes"/>
<interleave>
<zeroOrMore>
<ref name="atomAuthor"/>
</zeroOrMore>
<zeroOrMore>
<ref name="atomCategory"/>
</zeroOrMore>
<zeroOrMore>
<ref name="atomContributor"/>
</zeroOrMore>
<optional>
<ref name="atomGenerator"/>
</optional>
<optional>
<ref name="atomIcon"/>
</optional>
<optional>
<ref name="atomId"/>
</optional>
<zeroOrMore>
<ref name="atomLink"/>
</zeroOrMore>
<optional>
<ref name="atomLogo"/>
</optional>
<optional>
<ref name="atomRights"/>
</optional>
<optional>
<ref name="atomSubtitle"/>
</optional>
<optional>
<ref name="atomTitle"/>
</optional>
<optional>
<ref name="atomUpdated"/>
</optional>
<zeroOrMore>
<ref name="extensionElement"/>
</zeroOrMore>
</interleave>
</element>
</define>
<!-- atom:subtitle -->
<define name="atomSubtitle">
<element name="atom:subtitle">
<ref name="atomTextConstruct"/>
</element>
</define>
<!-- atom:summary -->
<define name="atomSummary">
<element name="atom:summary">
<ref name="atomTextConstruct"/>
</element>
</define>
<!-- atom:title -->
<define name="atomTitle">
<element name="atom:title">
<ref name="atomTextConstruct"/>
</element>
</define>
<!-- atom:updated -->
<define name="atomUpdated">
<element name="atom:updated">
<ref name="atomDateConstruct"/>
</element>
</define>
<!-- Low-level simple types -->
<define name="atomNCName">
<data type="string">
<param name="minLength">1</param>
<param name="pattern">[^:]*</param>
</data>
</define>
<!-- Whatever a media type is, it contains at least one slash -->
<define name="atomMediaType">
<data type="string">
<param name="pattern">.+/.+</param>
</data>
</define>
<!-- As defined in RFC 3066 -->
<define name="atomLanguageTag">
<data type="string">
<param name="pattern">[A-Za-z]{1,8}(-[A-Za-z0-9]{1,8})*</param>
</data>
</define>
<!--
Unconstrained; it's not entirely clear how IRI fit into
xsd:anyURI so let's not try to constrain it here
-->
<define name="atomUri">
<text/>
</define>
<!-- Whatever an email address is, it contains at least one @ -->
<define name="atomEmailAddress">
<data type="string">
<param name="pattern">.+@.+</param>
</data>
</define>
<!-- Simple Extension -->
<define name="simpleExtensionElement">
<element>
<anyName>
<except>
<nsName ns="http://www.w3.org/2005/Atom"/>
</except>
</anyName>
<text/>
</element>
</define>
<!-- Structured Extension -->
<define name="structuredExtensionElement">
<element>
<anyName>
<except>
<nsName ns="http://www.w3.org/2005/Atom"/>
</except>
</anyName>
<choice>
<group>
<oneOrMore>
<attribute>
<anyName/>
</attribute>
</oneOrMore>
<zeroOrMore>
<choice>
<text/>
<ref name="anyElement"/>
</choice>
</zeroOrMore>
</group>
<group>
<zeroOrMore>
<attribute>
<anyName/>
</attribute>
</zeroOrMore>
<group>
<optional>
<text/>
</optional>
<oneOrMore>
<ref name="anyElement"/>
</oneOrMore>
<zeroOrMore>
<choice>
<text/>
<ref name="anyElement"/>
</choice>
</zeroOrMore>
</group>
</group>
</choice>
</element>
</define>
<!-- Other Extensibility -->
<define name="extensionElement">
<choice>
<ref name="simpleExtensionElement"/>
<ref name="structuredExtensionElement"/>
</choice>
</define>
<define name="undefinedAttribute">
<attribute>
<anyName>
<except>
<name>xml:base</name>
<name>xml:lang</name>
<nsName ns=""/>
</except>
</anyName>
</attribute>
</define>
<define name="undefinedContent">
<zeroOrMore>
<choice>
<text/>
<ref name="anyForeignElement"/>
</choice>
</zeroOrMore>
</define>
<define name="anyElement">
<element>
<anyName/>
<zeroOrMore>
<choice>
<attribute>
<anyName/>
</attribute>
<text/>
<ref name="anyElement"/>
</choice>
</zeroOrMore>
</element>
</define>
<define name="anyForeignElement">
<element>
<anyName>
<except>
<nsName ns="http://www.w3.org/2005/Atom"/>
</except>
</anyName>
<zeroOrMore>
<choice>
<attribute>
<anyName/>
</attribute>
<text/>
<ref name="anyElement"/>
</choice>
</zeroOrMore>
</element>
</define>
<!-- XHTML -->
<define name="anyXHTML">
<element>
<nsName ns="http://www.w3.org/1999/xhtml"/>
<zeroOrMore>
<choice>
<attribute>
<anyName/>
</attribute>
<text/>
<ref name="anyXHTML"/>
</choice>
</zeroOrMore>
</element>
</define>
<define name="xhtmlDiv">
<element name="xhtml:div">
<zeroOrMore>
<choice>
<attribute>
<anyName/>
</attribute>
<text/>
<ref name="anyXHTML"/>
</choice>
</zeroOrMore>
</element>
</define>
</grammar>

View File

@ -1,28 +0,0 @@
<element name="limits" ns="http://docs.openstack.org/common/api/v1.0"
xmlns="http://relaxng.org/ns/structure/1.0">
<element name="rates">
<zeroOrMore>
<element name="rate">
<attribute name="uri"> <text/> </attribute>
<attribute name="regex"> <text/> </attribute>
<zeroOrMore>
<element name="limit">
<attribute name="value"> <text/> </attribute>
<attribute name="verb"> <text/> </attribute>
<attribute name="remaining"> <text/> </attribute>
<attribute name="unit"> <text/> </attribute>
<attribute name="next-available"> <text/> </attribute>
</element>
</zeroOrMore>
</element>
</zeroOrMore>
</element>
<element name="absolute">
<zeroOrMore>
<element name="limit">
<attribute name="name"> <text/> </attribute>
<attribute name="value"> <text/> </attribute>
</element>
</zeroOrMore>
</element>
</element>

View File

@ -185,7 +185,7 @@ class MethodInspector(object):
def build_polling_task(retriever, condition=lambda value: value,
sleep_time=1, time_out=0):
sleep_time=1, time_out=0, initial_delay=0):
"""Run a function in a loop with backoff on error.
The condition function runs based on the retriever function result.
@ -197,7 +197,8 @@ def build_polling_task(retriever, condition=lambda value: value,
raise loopingcall.LoopingCallDone(retvalue=obj)
call = loopingcall.BackOffLoopingCall(f=poll_and_check)
return call.start(initial_delay=0, starting_interval=sleep_time,
return call.start(initial_delay=initial_delay,
starting_interval=sleep_time,
max_interval=30, timeout=time_out)
@ -210,7 +211,7 @@ def wait_for_task(polling_task):
def poll_until(retriever, condition=lambda value: value,
sleep_time=3, time_out=0):
sleep_time=3, time_out=0, initial_delay=0):
"""Retrieves object until it passes condition, then returns it.
If time_out_limit is passed in, PollTimeOut will be raised once that
@ -218,7 +219,8 @@ def poll_until(retriever, condition=lambda value: value,
"""
task = build_polling_task(retriever, condition=condition,
sleep_time=sleep_time, time_out=time_out)
sleep_time=sleep_time, time_out=time_out,
initial_delay=initial_delay)
return wait_for_task(task)

View File

@ -218,8 +218,6 @@ class ConfigurationsController(wsgi.Controller):
def _refresh_on_all_instances(self, context, configuration_id):
"""Refresh a configuration group on all single instances.
"""
LOG.debug("Re-applying configuration group '%s' to all instances.",
configuration_id)
single_instances = instances_models.DBInstance.find_all(
tenant_id=context.project_id,
configuration_id=configuration_id,
@ -228,8 +226,8 @@ class ConfigurationsController(wsgi.Controller):
config = models.Configuration(context, configuration_id)
for dbinstance in single_instances:
LOG.debug("Re-applying configuration to instance: %s",
dbinstance.id)
LOG.info("Re-applying configuration %s to instance: %s",
configuration_id, dbinstance.id)
instance = instances_models.Instance.load(context, dbinstance.id)
instance.update_configuration(config)

View File

@ -314,7 +314,7 @@ class API(object):
device_path='/dev/vdb', mount_point='/mnt/volume',
backup_info=None, config_contents=None, root_password=None,
overrides=None, cluster_config=None, snapshot=None,
modules=None):
modules=None, ds_version=None):
"""Make an asynchronous call to prepare the guest
as a database container optionally includes a backup id for restores
"""
@ -335,7 +335,8 @@ class API(object):
device_path=device_path, mount_point=mount_point,
backup_info=backup_info, config_contents=config_contents,
root_password=root_password, overrides=overrides,
cluster_config=cluster_config, snapshot=snapshot, modules=modules)
cluster_config=cluster_config, snapshot=snapshot, modules=modules,
ds_version=ds_version)
def _create_guest_queue(self):
"""Call to construct, start and immediately stop rpc server in order
@ -409,15 +410,14 @@ class API(object):
self._call("reset_configuration", self.agent_high_timeout,
version=version, configuration=configuration)
def stop_db(self, do_not_start_on_reboot=False):
def stop_db(self):
"""Stop the database server."""
LOG.debug("Sending the call to stop the database process "
"on the Guest.")
version = self.API_BASE_VERSION
self._call("stop_db", self.agent_high_timeout,
version=version,
do_not_start_on_reboot=do_not_start_on_reboot)
self._call("stop_db", self.agent_low_timeout,
version=version)
def upgrade(self, instance_version, location, metadata=None):
"""Make an asynchronous call to self upgrade the guest agent."""

Some files were not shown because too many files have changed in this diff Show More