MNAIO: Add ability to deploy Ceph

This commit enables overriding the default storage options of
cinder/swift to a Ceph backed storage for the multinode all in one
deployments.

We also correct the README and build.sh script to show/use the
current defaults correctly for VM_DISK_SIZE and INFRA_VM_SERVER_RAM.

Change-Id: I9e1f1b09d1bcf224f4afa765c585baf28e6cafa8
This commit is contained in:
Jacob Wagner 2018-06-12 11:18:50 -05:00 committed by Jesse Pretorius
parent 58c7890bfc
commit 6b36fd8825
11 changed files with 356 additions and 18 deletions

View File

@ -107,12 +107,11 @@ to change this password please edit the pre-seed files.
Set an external inventory used for the MNAIO:
``MNAIO_INVENTORY=${MNAIO_INVENTORY:-playbooks/inventory}``
Set to instruct the preseed what the default network is expected to be:
``DEFAULT_NETWORK="${DEFAULT_NETWORK:-eth0}"``
Set the VM disk size in gigabytes:
``VM_DISK_SIZE="${VM_DISK_SIZE:-252}"``
``VM_DISK_SIZE="${VM_DISK_SIZE:-92160}"``
Instruct the system do all of the required host setup:
``SETUP_HOST=${SETUP_HOST:-true}``
@ -181,7 +180,7 @@ Instruct the system to use a set amount of ram for compute VM type:
``COMPUTE_VM_SERVER_RAM=${COMPUTE_VM_SERVER_RAM:-8196}``
Instruct the system to use a set amount of ram for infra VM type:
``INFRA_VM_SERVER_RAM=${INFRA_VM_SERVER_RAM:-8196}``
``INFRA_VM_SERVER_RAM=${INFRA_VM_SERVER_RAM:-16384}``
Instruct the system to use a set amount of ram for load balancer VM type:
``LOADBALANCER_VM_SERVER_RAM=${LOADBALANCER_VM_SERVER_RAM:-1024}``
@ -198,6 +197,8 @@ Instruct the system where to obtain iPXE kernels (looks for ipxe.lkrn, ipxe.efi,
Instruct the system to use a customized iPXE script during boot of VMs:
``IPXE_PATH_URL=${IPXE_PATH_URL:-''}``
Instruct the system to use CEPH block & object storage instead of the default LVM/swift:
``ENABLE_CEPH_STORAGE=${ENABLE_CEPH_STORAGE:-false}``
Re-kicking VM(s)
----------------
@ -296,3 +297,11 @@ available via a URL, they can be downloaded and used on a fresh host as follows.
# Then kick off the VM's from those images
ansible-playbook ${ANSIBLE_PARAMETERS} playbooks/deploy-vms.yml
Using Ceph-backed Block and Object Storage
------------------------------------------
To make use of Ceph in the environment, set ``ENABLE_CEPH_STORAGE`` to
``true``. This will disable the use of Swift as the Object Storage back-end
and disable the use of LVM as the Block Storage back-end, replacing both of
these with Ceph services.

View File

@ -26,6 +26,9 @@ ansible vm_hosts \
export MNAIO_ANSIBLE_PARAMETERS=${MNAIO_ANSIBLE_PARAMETERS:-""}
# TODO(odyssey4me):
# Replace this build override mechanism to just respect the
# group_vars instead of duplicating defaults here.
ansible-playbook -vv \
-i ${MNAIO_INVENTORY:-"playbooks/inventory"} \
-e setup_host=${SETUP_HOST:-"true"} \
@ -52,10 +55,11 @@ ansible-playbook -vv \
-e default_ubuntu_mirror_directory=${DEFAULT_MIRROR_DIR:-"/ubuntu"} \
-e cinder_vm_server_ram=${CINDER_VM_SERVER_RAM:-"2048"} \
-e compute_vm_server_ram=${COMPUTE_VM_SERVER_RAM:-"8196"} \
-e infra_vm_server_ram=${INFRA_VM_SERVER_RAM:-"8196"} \
-e infra_vm_server_ram=${INFRA_VM_SERVER_RAM:-"16384"} \
-e loadbalancer_vm_server_ram=${LOADBALANCER_VM_SERVER_RAM:-"2048"} \
-e logging_vm_server_ram=${LOGGING_VM_SERVER_RAM:-"2048"} \
-e swift_vm_server_ram=${SWIFT_VM_SERVER_RAM:-"2048"} \
-e enable_ceph_storage=${ENABLE_CEPH_STORAGE=-"false"} \
-e container_tech=${CONTAINER_TECH:-"lxc"} \
-e ipxe_kernel_base_url=${IPXE_KERNEL_BASE_URL:-"http://boot.ipxe.org"} \
-e ipxe_path_url=${IPXE_PATH_URL:-""} ${MNAIO_ANSIBLE_PARAMETERS} \

View File

@ -177,6 +177,8 @@
copy:
content: "{{ item.1.get_xml }}"
dest: "/etc/libvirt/qemu/{{ item.1.item }}.xml"
when:
- server_vm | default(false) | bool
delegate_to: "{{ item.0 }}"
with_nested:
- "{{ groups['vm_hosts'] }}"
@ -309,8 +311,8 @@
state: restarted
- name: Make space for swift and cinder volumes
hosts: cinder_hosts:swift_hosts
- name: Make space for swift/cinder/ceph volumes
hosts: cinder_hosts:swift_hosts:ceph_hosts:&vm_servers
gather_facts: "{{ gather_facts | default(true) }}"
environment: "{{ deployment_environment_variables | default({}) }}"
tags:
@ -348,27 +350,27 @@
_force_format_disks: "{{ (_remove_mounts is changed) or (_remove_lvs is changed) }}"
- name: Setup cinder host volume
hosts: cinder_hosts
hosts: cinder_hosts:&vm_servers
gather_facts: "{{ gather_facts | default(true) }}"
environment: "{{ deployment_environment_variables | default({}) }}"
tags:
- deploy-vms
tasks:
- name: Create cinder-volumes lv
- name: Create cinder-volumes LV
lvol:
vg: vmvg00
lv: cinder-volumes00
size: "100%FREE"
shrink: false
- name: Create data cinder-volumes group
- name: Create data cinder-volumes VG
lvg:
vg: cinder-volumes
pvs: "/dev/vmvg00/cinder-volumes00"
- name: Setup swift host volume
hosts: swift_hosts
hosts: swift_hosts:&vm_servers
gather_facts: "{{ gather_facts | default(true) }}"
environment: "{{ deployment_environment_variables | default({}) }}"
tags:
@ -405,3 +407,30 @@
- disk1
- disk2
- disk3
- name: Setup ceph OSD volumes
hosts: ceph_hosts:&vm_servers
gather_facts: "{{ gather_facts | default(true) }}"
environment: "{{ deployment_environment_variables | default({}) }}"
tags:
- deploy-vms
tasks:
- name: Create ceph OSD journal LV's
lvol:
vg: vmvg00
lv: "{{ item }}"
size: "{{ ceph_journal_size }}"
with_items:
- journal1
- journal2
- journal3
- name: Create ceph OSD disk LV's
lvol:
vg: vmvg00
lv: "{{ item }}"
size: "{{ ceph_osds_size }}"
with_items:
- data1
- data2
- data3

View File

@ -147,7 +147,7 @@ mnaio_host_networks:
osa_enable_infra: true
osa_enable_identity: true
osa_enable_block_storage: true
osa_enable_block_storage: "{{ not (enable_ceph_storage | bool) }}"
osa_enable_image: true
osa_enable_lbaas: false
osa_enable_compute: true
@ -155,9 +155,14 @@ osa_enable_orchestration: true
osa_enable_dashboard: true
osa_enable_network: true
osa_enable_meter: false
osa_enable_object_storage: true
osa_enable_object_storage: "{{ not (enable_ceph_storage | bool) }}"
osa_enable_legacy_os_infra: "{{ (osa_enable_image | bool) and (osa_enable_orchestration | bool) and (osa_enable_dashboard | bool) and (osa_enable_compute | bool) }}"
osa_disable_serial: false
osa_enable_elk_metrics: false
osa_enable_os_profiler: false
osa_enable_uwsgi_stats: false
# Ceph
enable_ceph_storage: false
ceph_journal_size: 5120
ceph_osds_filesystem_type: 'xfs'

View File

@ -0,0 +1,30 @@
---
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in witing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ansible_os_family: "{{ images[default_vm_image]['image_type'] }}"
server_domain_name: 'openstack.local'
server_vm: "{{ enable_ceph_storage | bool }}"
server_vm_ram: '{{ ceph_vm_server_ram | default(1024) }}'
server_vm_vcpus: '{{ ceph_vm_server_vcpus | default(2) }}'
server_vm_primary_network: 'dhcp'
server_image: "{{ default_vm_image }}"
server_default_interface: 'eth0'
server_preseed_ks: 'vm'
server_extra_options: ''
ceph_osds_size: 20480
default_vm_storage: "{{ ceph_vm_disk_size | default(122880) }}"

View File

@ -0,0 +1,50 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in witing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ansible_os_family: "{{ images[default_vm_image]['image_type'] }}"
server_hostname: 'ceph1'
server_vm_fixed_addr: '10.0.2.145'
server_mac_address: '52:54:00:bd:80:13'
server_networks:
dhcp:
iface: 'eth0'
inet_type: 'dhcp'
vm_int_iface: vm-br-dhcp
mgmt:
iface: 'eth1'
inet_type: 'static'
address: '10.0.236.145/22'
vm_int_iface: vm-br-eth1
flat:
iface: 'eth2'
inet_type: 'static'
address: '10.0.248.145/22'
vm_int_iface: vm-br-eth2
vlan:
iface: 'eth3'
inet_type: 'manual'
vm_int_iface: vm-br-eth3
vxlan:
iface: 'eth4'
inet_type: 'static'
address: '10.0.240.145/22'
vm_int_iface: vm-br-eth4
storage:
iface: 'eth5'
inet_type: 'static'
address: '10.0.244.145/22'
vm_int_iface: vm-br-eth5

View File

@ -0,0 +1,50 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in witing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ansible_os_family: "{{ images[default_vm_image]['image_type'] }}"
server_hostname: 'ceph2'
server_vm_fixed_addr: '10.0.2.161'
server_mac_address: '52:54:00:bd:80:14'
server_networks:
dhcp:
iface: 'eth0'
inet_type: 'dhcp'
vm_int_iface: vm-br-dhcp
mgmt:
iface: 'eth1'
inet_type: 'static'
address: '10.0.236.146/22'
vm_int_iface: vm-br-eth1
flat:
iface: 'eth2'
inet_type: 'static'
address: '10.0.248.146/22'
vm_int_iface: vm-br-eth2
vlan:
iface: 'eth3'
inet_type: 'manual'
vm_int_iface: vm-br-eth3
vxlan:
iface: 'eth4'
inet_type: 'static'
address: '10.0.240.146/22'
vm_int_iface: vm-br-eth4
storage:
iface: 'eth5'
inet_type: 'static'
address: '10.0.244.146/22'
vm_int_iface: vm-br-eth5

View File

@ -0,0 +1,50 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in witing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ansible_os_family: "{{ images[default_vm_image]['image_type'] }}"
server_hostname: 'ceph3'
server_vm_fixed_addr: '10.0.2.162'
server_mac_address: '52:54:00:bd:80:15'
server_networks:
dhcp:
iface: 'eth0'
inet_type: 'dhcp'
vm_int_iface: vm-br-dhcp
mgmt:
iface: 'eth1'
inet_type: 'static'
address: '10.0.236.147/22'
vm_int_iface: vm-br-eth1
flat:
iface: 'eth2'
inet_type: 'static'
address: '10.0.248.147/22'
vm_int_iface: vm-br-eth2
vlan:
iface: 'eth3'
inet_type: 'manual'
vm_int_iface: vm-br-eth3
vxlan:
iface: 'eth4'
inet_type: 'static'
address: '10.0.240.147/22'
vm_int_iface: vm-br-eth4
storage:
iface: 'eth5'
inet_type: 'static'
address: '10.0.244.147/22'
vm_int_iface: vm-br-eth5

View File

@ -10,6 +10,9 @@ mnaio1 ansible_host=10.0.2.1 ansible_connection=local
infra1 ansible_host=10.0.236.100
infra2 ansible_host=10.0.236.101
infra3 ansible_host=10.0.236.102
ceph1 ansible_host=10.0.236.145
ceph2 ansible_host=10.0.236.146
ceph3 ansible_host=10.0.236.147
cinder1 ansible_host=10.0.236.130
cinder2 ansible_host=10.0.236.131
compute1 ansible_host=10.0.236.120
@ -41,6 +44,9 @@ mnaio1
# The group "pxe_servers" is used to create PXE specific configurations for
# each host machines.
[pxe_servers]
ceph1
ceph2
ceph3
cinder1
cinder2
compute1
@ -94,3 +100,33 @@ cinder2
[compute_hosts]
compute1
compute2
# The group "ceph_hosts" is used in the MNAIO setup to create the basic drive
# setup for a ceph deployment
[ceph_hosts]
ceph1
ceph2
ceph3
######################## CEPH-ANSIBLE RELATED GROUPINGS ########################
# Used by ceph-ansible to determine which hosts to deploy to
[ceph]
ceph1 monitor_address=10.0.244.145
ceph2 monitor_address=10.0.244.146
ceph3 monitor_address=10.0.244.147
# Where to install the ceph monitoring service for ceph-ansible
[mons:children]
infra_hosts
# Where to install the osd services for ceph-ansible
[osds:children]
ceph
# Where to install the radosgw services for ceph-ansible
[rgws:children]
ceph
# Where to install the ceph metadata services if needed for ceph-ansible
[mdss]

View File

@ -70,7 +70,12 @@ global_overrides:
- cinder_api
- cinder_volume
- nova_compute
{% if (osa_enable_object_storage | bool) and not (enable_ceph_storage | bool) %}
- swift_proxy
{% endif %}
{% if enable_ceph_storage | bool %}
- ceph-osd
{% endif %}
- network:
container_bridge: "br-lbaas"
container_type: "veth"
@ -83,7 +88,7 @@ global_overrides:
- "octavia-worker"
- "octavia-housekeeping"
- "octavia-health-monitor"
{% if osa_enable_object_storage | bool %}
{% if (osa_enable_object_storage | bool) and not (enable_ceph_storage | bool) %}
swift:
part_power: 8
storage_network: 'br-storage'
@ -103,7 +108,7 @@ global_overrides:
###
### Anchors
###
{% if osa_enable_block_storage | bool %}
{% if (osa_enable_block_storage | bool) and not (enable_ceph_storage | bool) %}
cinder_block: &cinder_block
{% for host in groups['cinder_hosts'] %}
{{ hostvars[host]['server_hostname'] }}:
@ -159,7 +164,7 @@ log_block: &log_block
{% endfor %}
{% if osa_enable_object_storage | bool %}
{% if (osa_enable_object_storage | bool) and not (enable_ceph_storage | bool) %}
swift_block: &swift_block
{% for host in groups['swift_hosts'] %}
{{ hostvars[host]['server_hostname'] }}:
@ -170,6 +175,30 @@ swift_block: &swift_block
{% endif %}
{% if enable_ceph_storage | bool %}
ceph_osds_block: &ceph_osds_block
{% for host in groups['ceph_hosts'] %}
{{ hostvars[host]['server_hostname'] }}:
ip: {{ hostvars[host]['server_networks']['mgmt']['address'].split('/')[0] }}
container_vars:
container_tech: "{{ default_container_tech }}"
lvm_volumes:
- data: data1
data_vg: vmvg00
journal: journal1
journal_vg: vmvg00
- data: data2
data_vg: vmvg00
journal: journal2
journal_vg: vmvg00
- data: data3
data_vg: vmvg00
journal: journal3
journal_vg: vmvg00
{% endfor %}
{% endif %}
###
### Infrastructure
###
@ -188,6 +217,17 @@ log_hosts: *log_block
haproxy_hosts: *loadbalancer_block
{% endif %}
{% if enable_ceph_storage | bool %}
# Ceph Mon Hosts
ceph-mon_hosts: *infra_block
# Ceph RGW Hosts
ceph-rgw_hosts: *infra_block
# Ceph OSDs Hosts
ceph-osd_hosts: *ceph_osds_block
{% endif %}
###
### OpenStack
###
@ -203,13 +243,15 @@ os-infra_hosts: *infra_block
identity_hosts: *infra_block
{% endif %}
{% if osa_enable_block_storage | bool %}
{% if (osa_enable_block_storage | bool) or (enable_ceph_storage | bool) %}
# cinder api services
storage-infra_hosts: *infra_block
{% if not (enable_ceph_storage | bool) %}
# cinder storage host (LVM-backed)
storage_hosts: *cinder_block
{% endif %}
{% endif %}
{% if osa_enable_image | bool %}
# glance
@ -258,7 +300,7 @@ metrics_hosts: *infra_block
metering-compute_hosts: *compute_block
{% endif %}
{% if osa_enable_object_storage | bool %}
{% if (osa_enable_object_storage | bool) and not (enable_ceph_storage | bool) %}
# swift storage hosts
swift_hosts: *swift_block

View File

@ -1,5 +1,4 @@
---
# Tempest is being configured to use a known network
tempest_public_subnet_cidr: 10.0.248.0/26
@ -40,6 +39,40 @@ glance_glance_api_conf_overrides:
galera_innodb_buffer_pool_size: 1024M
galera_gcache_size: 128M
{% if enable_ceph_storage | bool %}
### Ceph Config Overrides
## Ceph cluster fsid (must be generated before first run)
generate_fsid: true
## ceph-ansible settings
## See https://github.com/ceph/ceph-ansible/tree/master/group_vars for
## additional configuration options availble.
monitor_address_block: 10.0.236.0/22
public_network: 10.0.236.0/22
cluster_network: 10.0.244.0/22
osd_scenario: lvm
journal_size: {{ ceph_journal_size }}
# ceph-ansible automatically creates pools & keys for OpenStack services
openstack_config: true
cinder_ceph_client: cinder
glance_ceph_client: glance
glance_default_store: rbd
glance_rbd_store_pool: images
nova_libvirt_images_rbd_pool: vms
{% raw %}
cinder_backends:
ceph:
volume_driver: cinder.volume.drivers.rbd.RBDDriver
rbd_pool: volumes
rbd_ceph_conf: /etc/ceph/ceph.conf
rbd_store_chunk_size: 8
volume_backend_name: rbddriver
rbd_user: "{{ cinder_ceph_client }}"
rbd_secret_uuid: "{{ cinder_ceph_client_uuid }}"
report_discard_supported: true
{% endraw %}
{% endif %}
{% if osa_enable_elk_metrics | bool %}
# NOTE(d34dh0r53): The disk detection to determine the logstash_queue_type
# can fail spectacularly on an MNAIO so I'm hard setting it here to bypass