Implement trove networking

To operate properly the trove guest agent needs access to
rabbitmq and also the neutron network for trove to use must
be created and defined in the trove.conf file.

This changeset adds documentation, tasks and
configuration defaults to setup the networking for
trove.

Change-Id: Idcf87c2eef0af475c02412f03433d22d7b08643f
This commit is contained in:
Kyle L. Henderson 2017-04-11 10:28:20 -05:00
parent 086bd308de
commit 9d8a9347c4
10 changed files with 336 additions and 117 deletions

View File

@ -1,5 +1,6 @@
---
# Copyright 2016 Internet Solutions (Pty) Ltd
# Copyright 2017 IBM Corp
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -54,10 +55,15 @@ trove_service_publicurl: "{{ trove_service_publicuri_proto }}://{{ external_lb_v
trove_service_internalurl: "{{ trove_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ trove_service_port }}/v1.0/%(tenant_id)s"
trove_service_adminurl: "{{ trove_service_adminuri_proto }}://{{ internal_lb_vip_address }}:{{ trove_service_port }}/v1.0/%(tenant_id)s"
trove_auth_url: "{{ keystone_service_internalurl }}"
# By default assume the trove service network is an internal one
# It would be good to use the catalog service to lookup these URLs based on the endpoint type
# but currently the Trove code doesn't work properly when endpoint types are specified
trove_nova_compute_url: "{{ trove_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ nova_service_port }}/v2.1"
trove_cinder_url: "{{ trove_service_publicuri_proto }}://{{ external_lb_vip_address }}:{{ cinder_service_port }}/v1"
trove_swift_url: "{{ trove_service_publicuri_proto }}://{{ external_lb_vip_address }}:{{ swift_proxy_port }}/v1/AUTH_"
trove_neutron_url: "{{ trove_service_publicuri_proto }}://{{ external_lb_vip_address }}:{{ neutron_service_port }}/"
trove_cinder_url: "{{ trove_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ cinder_service_port }}/v1"
trove_swift_url: "{{ trove_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ swift_proxy_port }}/v1/AUTH_"
trove_neutron_url: "{{ trove_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ neutron_service_port }}/"
trove_profiler_enabled: false
## Cap the maximum number of threads / workers when a user value is unspecified.
@ -125,6 +131,39 @@ trove_rabbitmq_use_ssl: False
trove_rabbitmq_port: 5672
trove_rabbitmq_servers: "{{ rabbitmq_servers }}"
# The trove guest agent in the deployed DB VMs need access to OpenStack services (keystone, swift, etc)
# and also to rabbitmq. The way it gets access (networking) these services may differ.
# By default assume:
# OpenStack services are accesses through public interfaces
# Infrastructure services (rabbitmq) are accessed through the defined provider network.
#
# The value of 'net_name' field of the provider network network to use for infrastructure services
trove_provider_net_name: dbaas-mgmt
trove_provider_network: "{{ provider_networks|map(attribute='network')|selectattr('net_name','defined')|selectattr('net_name', 'equalto', trove_provider_net_name)|list|first }}"
# The value of 'ip_from_q' field
trove_provider_ip_from_q: "{{ trove_provider_network['ip_from_q'] }}"
# The name of the network address pool
trove_container_net_name: "{{ trove_provider_ip_from_q }}_address"
trove_guest_rabbitmq_servers: "{% for host in groups[rabbitmq_host_group] %}{{ hostvars[host]['container_networks'][trove_container_net_name]['address'] }}{% if not loop.last %},{% endif %}{% endfor %}"
trove_guest_rabbitmq_use_ssl: "{{ trove_rabbitmq_use_ssl }}"
# For OpenStack services that have public, admin, and internal access, use the public ones for the guest VMs.
trove_guest_auth_url: "{{ keystone_service_publicurl }}"
trove_guest_swift_url: "{{ trove_service_publicuri_proto }}://{{ external_lb_vip_address }}:{{ swift_proxy_port }}/v1/AUTH_"
# Trove service network settings.
# These values are used when creating an OpenStack network to be used by Trove. By default the network will
# not be created.
trove_service_net_setup: False
trove_service_net_validate_certs: "False"
trove_service_net_phys_net: dbaas-mgmt
trove_service_net_type: flat
trove_service_net_name: dbaas_service_net
trove_service_subnet_name: dbaas_service_subnet
trove_service_net_subnet_cidr: "192.168.20.0/24"
trove_service_net_dhcp: "True"
trove_service_net_allocation_pool_start: "192.168.20.100"
trove_service_net_allocation_pool_end: "192.168.20.120"
# RPC encryption keys
# See the Trove documentation as to the significance of the rpc encryption keys
# Trove supplies default values but we enforce they not be left to their default values
@ -158,6 +197,8 @@ trove_requires_pip_packages:
- python-keystoneclient
- virtualenv
- virtualenv-tools
- shade
- futures
trove_pip_packages:
- MySQL-python
@ -167,6 +208,7 @@ trove_pip_packages:
- trove
- python-troveclient
- python-memcached
- os-client-config
# This variable is used by the repo_build process to determine
# which host group to check for members of before building the

View File

@ -0,0 +1,105 @@
=================
Configuring Trove
=================
.. note::
Care should be taken when deploying Trove in production environments.
Be sure to fully understand the security implications of the deployed
architecture.
Trove provides DBaaS to an OpenStack deployment. It deploys guest VMs that
provide the desired DB for use by the end consumer. The trove guest VMs need
connectivity back to the trove services via RPC (rabbitmq) and the OpenStack
services. The way these guest VM get access to those services could be via
internal networking (in the case of rabbitmq) or via public interfaces (in the
case of OpenStack services). For the example configuration, we'll designate
a provider network as the network for trove to provision on each guest VM. The
guest can then connect to rabbitmq via this network and to the OpenStack
services externally. Optionally, the guest VMs could use the internal network
to access OpenStack services, but that would require more containers being
bound to this network.
The deployment configuration outlined below may not be appropriate for
production environments. Review this very carefully with your own security
requirements.
Setup a neutron network for use by trove
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Trove needs connectivity between the control plane and the DB guest VMs. For
this purpose a provider network should be created which bridges the trove
containers (if the control plane is installed in a container) or hosts with
VMs. In a general case, neutron networking can be a simple flat network.
An example entry into ``openstack_user_config.yml`` is shown below:
.. code-block:: yaml
- network:
container_bridge: "br-dbaas"
container_type: "veth"
container_interface: "eth14"
host_bind_override: "eth14"
ip_from_q: "dbaas"
type: "flat"
net_name: "dbaas-mgmt"
group_binds:
- neutron_linuxbridge_agent
- rabbitmq
Make sure to modify the other entries in this file as well.
The ``net_name`` will be the physical network that is specified when creating
the neutron network. The default value of ``dbaas-mgmt`` is also used to
lookup the addresses of the rabbitmq container. If the default is not used then
some variables in ``defaults\main.yml`` will need to be overwritten.
By default this role will not create the neutron network automaticaly. However,
the default values can be changed to create the neutron network. See the
``trove_service_net_*`` variable in ``defaults\main.yml``. By customizing the
``trove_service_net_*`` variables and having this role create the neutron
network a full deployment of the OpenStack and DBaaS can proceed
without interruption or intervention.
The following is an example how to set up a provider network in neutron
manually, if so desired:
.. code-block:: bash
neutron net-create dbaas_service_net --shared \
--provider:network_type flat \
--provider:physical_network dbaas-mgmt
neutron subnet-create dbaas_service_net 172.19.0.0/22 --name dbaas_service_subnet
--ip-version=4 \
--allocation-pool start=172.19.1.100,end=172.19.1.200 \
--enable-dhcp \
--dns-nameservers list=true 8.8.4.4 8.8.8.8
Special attention needs to be applied to the ``--allocation-pool`` to not have
ips which overlap with ips assigned to hosts or containers (see the ``used_ips``
variable in ``openstack_user_config.yml``)
.. note::
This role needs the neutron network created before it can run properly
since the trove guest agent configuration file contains that information.
Building Trove images
~~~~~~~~~~~~~~~~~~~~~
When building disk image for the guest VM deployments there are many items
to consider. Listed below are a few:
#. Security of the VM and the network infrastructure
#. What DBs will be installed
#. What DB services will be supported
#. How will the images be maintained
Images can be built using the ``diskimage-builder`` tooling. The trove
virtual environment can be tar'd up from the trove containers and deployed to
the images using custom ``diskimage-builder`` elements.
See the ``trove/integration/scripts/files/elements`` directory contents in
the OpenStack Trove project for ``diskimage-builder`` elements to build trove
disk images.

View File

@ -7,6 +7,11 @@ This Ansible role installs Trove.
To clone or view the source code for this repository, visit the role repository
for `os_trove <https://github.com/openstack/openstack-ansible-os_trove>`_.
.. toctree::
:maxdepth: 2
configure-trove.rst
Default variables
~~~~~~~~~~~~~~~~~

View File

@ -43,6 +43,11 @@
tags:
- trove-install
- include: trove_service_network.yml
tags:
- trove-install
- trove-config
- include: trove_post_install.yml
tags:
- trove-install

View File

@ -0,0 +1,77 @@
---
# Copyright 2016,2017 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- block:
- name: Get admin tenant id
keystone:
command: get_tenant
tenant_name: admin
endpoint: "{{ keystone_service_adminurl }}"
login_user: "{{ keystone_admin_user_name }}"
login_password: "{{ keystone_auth_admin_password }}"
login_project_name: "{{ keystone_admin_tenant_name }}"
- name: Store admin tenant id
set_fact:
keystone_admin_tenant_id: "{{ keystone_facts.id }}"
- name: Create trove service network
os_network:
cloud: default
validate_certs: "{{ trove_service_net_validate_certs }}"
state: present
name: "{{ trove_service_net_name }}"
provider_physical_network: "{{ trove_service_net_phys_net }}"
provider_network_type: "{{ trove_service_net_type }}"
project: "{{ keystone_admin_tenant_id }}"
wait: yes
register: trove_network
run_once: true
- name: Create trove service subnet
os_subnet:
cloud: default
validate_certs: "{{ trove_service_net_validate_certs }}"
state: present
network_name: "{{ trove_service_net_name }}"
name: "{{ trove_service_subnet_name }}"
allocation_pool_start: "{{ trove_service_net_allocation_pool_start | default(omit) }}"
allocation_pool_end: "{{ trove_service_net_allocation_pool_end | default(omit) }}"
cidr: "{{ trove_service_net_subnet_cidr }}"
enable_dhcp: "{{ trove_service_net_dhcp }}"
project: "{{ keystone_admin_tenant_id }}"
wait: yes
register: subnet_create
run_once: true
# Block end
when: trove_service_net_setup
# At this point we need to get the neutron network for trove to use. In our role testing
# environment this won't succeed. So we'll bypass it when in developer mode otherwise,
# this task will fail if trove_service_net_setup is false or else if the
# network isn't pre-created by the user.
- name: Get trove service net id
os_networks_facts:
cloud: default
validate_certs: "{{ trove_service_net_validate_certs }}"
wait: yes
name: "{{ trove_service_net_name }}"
run_once: true
when: not trove_developer_mode | bool
- name: Save trove service net id
set_fact:
trove_service_net_id: "{% if openstack_networks is defined %}{{ openstack_networks[0].id }}{% else %}UNKNOWN{% endif %}"
run_once: true

41
templates/include_db.j2 Normal file
View File

@ -0,0 +1,41 @@
[mysql]
root_on_create = False
tcp_ports = 3306
volume_support = True
device_path = /dev/vdb
# Users to ignore for user create/list/delete operations
ignore_users = os_admin, root
ignore_dbs = mysql, information_schema, performance_schema
[redis]
tcp_ports = 6379
#redis uses local storage
volume_support = True
# default device_path = None
[cassandra]
tcp_ports = 7000, 7001, 9042, 9160
volume_support = True
device_path = /dev/vdb
[couchbase]
tcp_ports = 8091, 8092, 4369, 11209-11211, 21100-21199
volume_support = True
device_path = /dev/vdb
[mongodb]
tcp_ports = 2500, 27017
volume_support = True
device_path = /dev/vdb
num_config_servers_per_cluster = 1
num_query_routers_per_cluster = 1
[vertica]
tcp_ports = 5433, 5434, 22, 5444, 5450, 4803
udp_ports = 5433, 4803, 4804, 6453
volume_support = True
device_path = /dev/vdb
cluster_support = True
cluster_member_count = 3
api_strategy = trove.common.strategies.cluster.experimental.vertica.api.VerticaAPIStrategy

View File

@ -2,13 +2,13 @@
[DEFAULT]
rpc_backend = {{ trove_rpc_backend }}
control_exchange = {{ trove_control_exchange }}
trove_auth_url = {{ trove_auth_url }}
trove_auth_url = {{ trove_guest_auth_url }}
nova_proxy_admin_user = {{ trove_service_user_name }}
nova_proxy_admin_pass = {{ trove_service_password }}
nova_proxy_admin_tenant_name = {{ trove_service_project_name }}
{% if trove_swift_enabled is defined %}
swift_url = {{ trove_swift_url }}
swift_url = {{ trove_guest_swift_url }}
# Region name of this node. Default value is None.
os_region_name = {{ trove_service_region }}
# Service type to use when searching catalog.
@ -24,8 +24,8 @@ datastore_registry_ext = mysql:trove.guestagent.datastore.mysql.manager.Manager,
root_grant = ALL
root_grant_option = True
# For storage to Swift, use the following as defaults:
{% if trove_storage_strategy is defined %}
# For storage to Swift, use the following as defaults:
storage_strategy = SwiftStorage
storage_namespace = trove.common.strategies.storage.swift
@ -53,21 +53,23 @@ log_file = trove-guestagent.log
#trace_sqlalchemy = True
[oslo_messaging_rabbit]
{% if trove_guest_rabbitmq_servers.split(',') | length > 1 %}
# RabbitMQ HA cluster host:port pairs. (list value)
# Deprecated group/name - [DEFAULT]/rabbit_hosts
rabbit_hosts={% for host in trove_guest_rabbitmq_servers.split(',') %}{{ host }}:{{ trove_rabbitmq_port }}{% if not loop.last %},{% endif %}{% endfor %}
{% else %}
# The RabbitMQ broker address where a single node is used. (string value)
# Deprecated group/name - [DEFAULT]/rabbit_host
rabbit_host = {{ trove_rabbitmq_servers }}
rabbit_host = {{ trove_guest_rabbitmq_servers }}
# The RabbitMQ broker port where a single node is used. (integer value)
# Deprecated group/name - [DEFAULT]/rabbit_port
rabbit_port= {{ trove_rabbitmq_port }}
# RabbitMQ HA cluster host:port pairs. (list value)
# Deprecated group/name - [DEFAULT]/rabbit_hosts
#rabbit_hosts=$rabbit_host:$rabbit_port
rabbit_port = {{ trove_rabbitmq_port }}
{% endif %}
# Connect over SSL for RabbitMQ. (boolean value)
# Deprecated group/name - [DEFAULT]/rabbit_use_ssl
#rabbit_use_ssl=false
rabbit_use_ssl = {{ trove_guest_rabbitmq_use_ssl }}
# The RabbitMQ userid. (string value)
# Deprecated group/name - [DEFAULT]/rabbit_userid

View File

@ -13,25 +13,44 @@ taskmanager_rpc_encr_key = {{ trove_taskmanager_rpc_encr_key }}
inst_rpc_key_encr_key = {{ trove_inst_rpc_key_encr_key }}
db_api_implementation = trove.db.sqlalchemy.api
trove_auth_url = {{ trove_auth_url }}
nova_compute_url = {{ trove_nova_compute_url }}
cinder_url = {{ trove_cinder_url }}
swift_url = {{ trove_swift_url }}
neutron_url = {{ trove_neutron_url }}
os_region_name = {{ trove_service_region }}
nova_compute_service_type = compute
cinder_service_type = volumev2
swift_service_type = object-store
heat_service_type = orchestration
neutron_service_type = network
trove_volume_support = True
block_device_mapping = vdb
device_path = /dev/vdb
mount_point = /var/lib/mysql
volume_time_out=30
server_delete_time_out=480
use_nova_server_config_drive = False
use_nova_server_config_drive = True
taskmanager_manager=trove.taskmanager.manager.Manager
# Keystone
trove_auth_url = {{ trove_auth_url }}
os_region_name = {{ trove_service_region }}
# Nova
nova_compute_url = {{ trove_nova_compute_url }}
nova_compute_service_type = compute
nova_proxy_admin_user = {{ trove_service_user_name }}
nova_proxy_admin_pass = {{ trove_service_password }}
nova_proxy_admin_tenant_name = {{ trove_service_project_name }}
# Cinder
cinder_url = {{ trove_cinder_url }}
cinder_service_type = volumev2
#Swift
swift_url = {{ trove_swift_url }}
swift_service_type = object-store
# Neutron
neutron_url = {{ trove_neutron_url }}
neutron_service_type = network
network_driver = trove.network.neutron.NeutronDriver
default_neutron_networks = {{ trove_service_net_id }}
# Heat
heat_service_type = orchestration
# Notifications
exists_notification_transformer = trove.extensions.mgmt.instances.models.NovaNotificationTransformer
exists_notification_ticks = 30
notification_service_id = mysql:2f3ff068-2bfb-4f70-9a9d-a6bb65bc084b
@ -50,16 +69,6 @@ dns_instance_entry_factory = trove.dns.designate.driver.DesignateInstanceEntryFa
dns_endpoint_url = http://127.0.0.1/v1/
dns_service_type = dns
# Neutron
network_driver = trove.network.nova.NovaNetwork
default_neutron_networks =
# Nova
nova_compute_url = {{ trove_nova_compute_url }}
nova_proxy_admin_user = {{ trove_service_user_name }}
nova_proxy_admin_pass = {{ trove_service_password }}
nova_proxy_admin_tenant_name = {{ trove_service_project_name }}
# Trove Security Groups for Instances
trove_security_groups_support = True
trove_security_group_rule_cidr = 0.0.0.0/0
@ -82,11 +91,9 @@ template_path = /etc/trove/templates/
pydev_debug = disabled
# ================= Guestagent related ========================
#guest_config = /etc/trove/trove-guestagent.conf
# Use 'guest_info = /etc/guest_info' for pre-Kilo compatibility
#guest_info = guest_info.conf
# Use 'injected_config_location = /etc/trove' for pre-Kilo compatibility
#injected_config_location = /etc/trove/conf.d
guest_config = /etc/trove/trove-guestagent.conf
guest_info = guest_info.conf
injected_config_location = /etc/trove/conf.d
#cloudinit_location = /etc/trove/cloudinit
[database]
@ -101,35 +108,4 @@ enabled = {{ trove_profiler_enabled }}
[oslo_messaging_rabbit]
rabbit_use_ssl={{ trove_rabbitmq_use_ssl }}
[mysql]
tcp_ports = 3306
volume_support = True
device_path = /dev/vdb
[redis]
tcp_ports = 6379
# redis uses local storage
volume_support = False
# default device_path = None
[cassandra]
tcp_ports = 7000, 7001, 9042, 9160
volume_support = True
device_path = /dev/vdb
[couchbase]
tcp_ports = 8091, 8092, 4369, 11209-11211, 21100-21199
volume_support = True
device_path = /dev/vdb
[mongodb]
volume_support = True
device_path = /dev/vdb
[vertica]
tcp_ports = 5433, 5434, 22, 5444, 5450, 4803
udp_ports = 5433, 4803, 4804, 6453
volume_support = True
device_path = /dev/vdb
mount_point = /var/lib/vertica
taskmanager_strategy = trove.common.strategies.cluster.experimental.vertica.taskmanager.VerticaTaskManagerStrategy
{% include 'include_db.j2' %}

View File

@ -57,8 +57,13 @@ nova_proxy_admin_pass = {{ trove_service_password }}
nova_proxy_admin_tenant_name = {{ trove_service_project_name }}
# Neutron
network_driver = trove.network.nova.NovaNetwork
default_neutron_networks =
network_driver = trove.network.neutron.NeutronDriver
default_neutron_networks = {{ trove_service_net_id }}
neutron_url = {{ trove_neutron_url }}
# Swift
swift_url = {{ trove_swift_url }}
swift_service_type = object-store
# Taskmanager queue name
taskmanager_queue = taskmanager
@ -106,53 +111,12 @@ enabled = {{ trove_profiler_enabled }}
#trace_sqlalchemy = True
[ssl]
#cert_file = /path/to/server.crt
#key_file = /path/to/server.key
#optional:
#ca_file = /path/to/ca_file
[oslo_messaging_rabbit]
rabbit_use_ssl={{ trove_rabbitmq_use_ssl }}
rabbit_use_ssl = {{ trove_rabbitmq_use_ssl }}
[mysql]
root_on_create = False
tcp_ports = 3306
volume_support = True
device_path = /dev/vdb
# Users to ignore for user create/list/delete operations
ignore_users = os_admin, root
ignore_dbs = mysql, information_schema, performance_schema
[redis]
tcp_ports = 6379
#redis uses local storage
volume_support = False
# default device_path = None
[cassandra]
tcp_ports = 7000, 7001, 9042, 9160
volume_support = True
device_path = /dev/vdb
[couchbase]
tcp_ports = 8091, 8092, 4369, 11209-11211, 21100-21199
volume_support = True
device_path = /dev/vdb
[mongodb]
tcp_ports = 2500, 27017
volume_support = True
device_path = /dev/vdb
num_config_servers_per_cluster = 1
num_query_routers_per_cluster = 1
[vertica]
tcp_ports = 5433, 5434, 22, 5444, 5450, 4803
udp_ports = 5433, 4803, 4804, 6453
volume_support = True
device_path = /dev/vdb
cluster_support = True
cluster_member_count = 3
api_strategy = trove.common.strategies.cluster.experimental.vertica.api.VerticaAPIStrategy
{% include 'include_db.j2' %}

View File

@ -27,6 +27,8 @@ trove_rabbitmq_use_ssl: "{{ rabbitmq_use_ssl }}"
trove_rabbitmq_password: "secrete"
trove_rabbitmq_userid: trove
trove_rabbitmq_vhost: /trove
trove_guest_rabbitmq_servers: "{{ trove_rabbitmq_servers }}"
trove_guest_auth_url: "{{ trove_auth_url }}"
trove_requirements_git_install_branch: master
trove_service_password: "secrete"
trove_regular_user_password: "secrete"