Merge "Support separate Swift storage networks"

This commit is contained in:
Zuul 2019-03-14 16:19:59 +00:00 committed by Gerrit Code Review
commit e35c32c0d7
18 changed files with 306 additions and 9 deletions

View File

@ -167,6 +167,8 @@ kolla_external_vip_interface: "{{ network_interface }}"
api_interface: "{{ network_interface }}"
storage_interface: "{{ network_interface }}"
cluster_interface: "{{ network_interface }}"
swift_storage_interface: "{{ storage_interface }}"
swift_replication_interface: "{{ swift_storage_interface }}"
migration_interface: "{{ network_interface }}"
tunnel_interface: "{{ network_interface }}"
octavia_network_interface: "{{ api_interface }}"

View File

@ -66,6 +66,9 @@ swift_devices_match_mode: "strict"
swift_devices_name: "KOLLA_SWIFT_DATA"
swift_delay_auth_decision: "False"
# Boolean, true if there is a dedicated replication network.
swift_has_replication_network: "{{ swift_storage_interface != swift_replication_interface }}"
openstack_swift_auth: "{{ openstack_auth }}"
syslog_server: "{{ api_interface_address }}"

View File

@ -11,14 +11,17 @@
- "swift"
- "swift-account-auditor"
- "swift-account-reaper"
- "swift-account-replication-server"
- "swift-account-replicator"
- "swift-account-server"
- "swift-container-auditor"
- "swift-container-replication-server"
- "swift-container-replicator"
- "swift-container-server"
- "swift-container-updater"
- "swift-object-auditor"
- "swift-object-expirer"
- "swift-object-replication-server"
- "swift-object-replicator"
- "swift-object-server"
- "swift-object-updater"
@ -34,14 +37,17 @@
with_items:
- "swift-account-auditor"
- "swift-account-reaper"
- "swift-account-replication-server"
- "swift-account-replicator"
- "swift-account-server"
- "swift-container-auditor"
- "swift-container-replication-server"
- "swift-container-replicator"
- "swift-container-server"
- "swift-container-updater"
- "swift-object-auditor"
- "swift-object-expirer"
- "swift-object-replication-server"
- "swift-object-replicator"
- "swift-object-server"
- "swift-object-updater"
@ -64,14 +70,17 @@
with_items:
- "account-auditor"
- "account-reaper"
- "account-replication-server"
- "account-replicator"
- "account-server"
- "container-auditor"
- "container-replication-server"
- "container-replicator"
- "container-server"
- "container-updater"
- "object-auditor"
- "object-expirer"
- "object-replication-server"
- "object-replicator"
- "object-server"
- "object-updater"
@ -93,6 +102,7 @@
with_items:
- "account-auditor"
- "account-reaper"
- "account-replication-server"
- "account-replicator"
- "account-server"
@ -111,6 +121,7 @@
become: true
with_items:
- "container-auditor"
- "container-replication-server"
- "container-replicator"
- "container-server"
- "container-updater"
@ -131,6 +142,7 @@
with_items:
- "object-auditor"
- "object-expirer"
- "object-replication-server"
- "object-replicator"
- "object-server"
- "object-updater"
@ -184,14 +196,17 @@
with_items:
- "swift-account-auditor"
- "swift-account-reaper"
- "swift-account-replication-server"
- "swift-account-replicator"
- "swift-account-server"
- "swift-container-auditor"
- "swift-container-replication-server"
- "swift-container-replicator"
- "swift-container-server"
- "swift-container-updater"
- "swift-object-auditor"
- "swift-object-expirer"
- "swift-object-replication-server"
- "swift-object-replicator"
- "swift-object-server"
- "swift-object-updater"

View File

@ -13,14 +13,17 @@
- "swift_rsyncd"
- "swift_account_server"
- "swift_account_auditor"
- "swift_account_replication_server"
- "swift_account_replicator"
- "swift_account_reaper"
- "swift_container_server"
- "swift_container_auditor"
- "swift_container_replication_server"
- "swift_container_replicator"
- "swift_container_updater"
- "swift_object_server"
- "swift_object_auditor"
- "swift_object_replication_server"
- "swift_object_replicator"
- "swift_object_updater"
- "swift_object_expirer"

View File

@ -10,7 +10,7 @@
- name: Checking free port for Swift Account Server
wait_for:
host: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
host: "{{ hostvars[inventory_hostname]['ansible_' + swift_storage_interface]['ipv4']['address'] }}"
port: "{{ swift_account_server_port }}"
connect_timeout: 1
timeout: 1
@ -21,7 +21,7 @@
- name: Checking free port for Swift Container Server
wait_for:
host: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
host: "{{ hostvars[inventory_hostname]['ansible_' + swift_storage_interface]['ipv4']['address'] }}"
port: "{{ swift_container_server_port }}"
connect_timeout: 1
timeout: 1
@ -32,7 +32,40 @@
- name: Checking free port for Swift Object Server
wait_for:
host: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
host: "{{ hostvars[inventory_hostname]['ansible_' + swift_storage_interface]['ipv4']['address'] }}"
port: "{{ swift_object_server_port }}"
connect_timeout: 1
timeout: 1
state: stopped
when:
- container_facts['swift_object_server'] is not defined
- inventory_hostname in groups['swift-object-server']
- name: Checking free port for Swift Account Replication Server
wait_for:
host: "{{ hostvars[inventory_hostname]['ansible_' + swift_replication_interface]['ipv4']['address'] }}"
port: "{{ swift_account_server_port }}"
connect_timeout: 1
timeout: 1
state: stopped
when:
- container_facts['swift_account_server'] is not defined
- inventory_hostname in groups['swift-account-server']
- name: Checking free port for Swift Container Replication Server
wait_for:
host: "{{ hostvars[inventory_hostname]['ansible_' + swift_replication_interface]['ipv4']['address'] }}"
port: "{{ swift_container_server_port }}"
connect_timeout: 1
timeout: 1
state: stopped
when:
- container_facts['swift_container_server'] is not defined
- inventory_hostname in groups['swift-container-server']
- name: Checking free port for Swift Object Replication Server
wait_for:
host: "{{ hostvars[inventory_hostname]['ansible_' + swift_replication_interface]['ipv4']['address'] }}"
port: "{{ swift_object_server_port }}"
connect_timeout: 1
timeout: 1
@ -43,7 +76,7 @@
- name: Checking free port for Rsync
wait_for:
host: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
host: "{{ hostvars[inventory_hostname]['ansible_' + swift_replication_interface]['ipv4']['address'] }}"
port: "873"
connect_timeout: 1
timeout: 1

View File

@ -10,16 +10,19 @@
with_items:
- { name: swift_account_server, group: swift-account-server }
- { name: swift_account_auditor, group: swift-account-server }
- { name: swift_account_replication_server, group: swift-account-server }
- { name: swift_account_replicator, group: swift-account-server }
- { name: swift_account_reaper, group: swift-account-server }
- { name: swift_rsyncd, group: swift-account-server }
- { name: swift_container_server, group: swift-container-server }
- { name: swift_container_auditor, group: swift-container-server }
- { name: swift_container_replication_server, group: swift-container-server }
- { name: swift_container_replicator, group: swift-container-server }
- { name: swift_container_updater, group: swift-container-server }
- { name: swift_rsyncd, group: swift-container-server }
- { name: swift_object_server, group: swift-object-server }
- { name: swift_object_auditor, group: swift-object-server }
- { name: swift_object_replication_server, group: swift-object-server }
- { name: swift_object_replicator, group: swift-object-server }
- { name: swift_object_updater, group: swift-object-server }
- { name: swift_object_expirer, group: swift-object-server }
@ -37,16 +40,19 @@
with_items:
- { name: swift_account_server, group: swift-account-server }
- { name: swift_account_auditor, group: swift-account-server }
- { name: swift_account_replication_server, group: swift-account-server }
- { name: swift_account_replicator, group: swift-account-server }
- { name: swift_account_reaper, group: swift-account-server }
- { name: swift_rsyncd, group: swift-account-server }
- { name: swift_container_server, group: swift-container-server }
- { name: swift_container_auditor, group: swift-container-server }
- { name: swift_container_replication_server, group: swift-container-server }
- { name: swift_container_replicator, group: swift-container-server }
- { name: swift_container_updater, group: swift-container-server }
- { name: swift_rsyncd, group: swift-container-server }
- { name: swift_object_server, group: swift-object-server }
- { name: swift_object_auditor, group: swift-object-server }
- { name: swift_object_replication_server, group: swift-object-server }
- { name: swift_object_replicator, group: swift-object-server }
- { name: swift_object_updater, group: swift-object-server }
- { name: swift_object_expirer, group: swift-object-server }
@ -66,16 +72,19 @@
with_items:
- { name: swift_account_server, group: swift-account-server }
- { name: swift_account_auditor, group: swift-account-server }
- { name: swift_account_replication_server, group: swift-account-server }
- { name: swift_account_replicator, group: swift-account-server }
- { name: swift_account_reaper, group: swift-account-server }
- { name: swift_rsyncd, group: swift-account-server }
- { name: swift_container_server, group: swift-container-server }
- { name: swift_container_auditor, group: swift-container-server }
- { name: swift_container_replication_server, group: swift-container-server }
- { name: swift_container_replicator, group: swift-container-server }
- { name: swift_container_updater, group: swift-container-server }
- { name: swift_rsyncd, group: swift-container-server }
- { name: swift_object_server, group: swift-object-server }
- { name: swift_object_auditor, group: swift-object-server }
- { name: swift_object_replication_server, group: swift-object-server }
- { name: swift_object_replicator, group: swift-object-server }
- { name: swift_object_updater, group: swift-object-server }
- { name: swift_object_expirer, group: swift-object-server }
@ -95,16 +104,19 @@
with_together:
- [{ name: swift_account_server, group: swift-account-server },
{ name: swift_account_auditor, group: swift-account-server },
{ name: swift_account_replication_server, group: swift-account-server },
{ name: swift_account_replicator, group: swift-account-server },
{ name: swift_account_reaper, group: swift-account-server },
{ name: swift_rsyncd, group: swift-account-server },
{ name: swift_container_server, group: swift-container-server },
{ name: swift_container_auditor, group: swift-container-server },
{ name: swift_container_replication_server, group: swift-container-server },
{ name: swift_container_replicator, group: swift-container-server },
{ name: swift_container_updater, group: swift-container-server },
{ name: swift_rsyncd, group: swift-container-server },
{ name: swift_object_server, group: swift-object-server },
{ name: swift_object_auditor, group: swift-object-server },
{ name: swift_object_replication_server, group: swift-object-server },
{ name: swift_object_replicator, group: swift-object-server },
{ name: swift_object_updater, group: swift-object-server },
{ name: swift_object_expirer, group: swift-object-server },
@ -131,16 +143,19 @@
with_together:
- [{ name: swift_account_server, group: swift-account-server },
{ name: swift_account_auditor, group: swift-account-server },
{ name: swift_account_replication_server, group: swift-account-server },
{ name: swift_account_replicator, group: swift-account-server },
{ name: swift_account_reaper, group: swift-account-server },
{ name: swift_rsyncd, group: swift-account-server },
{ name: swift_container_server, group: swift-container-server },
{ name: swift_container_auditor, group: swift-container-server },
{ name: swift_container_replication_server, group: swift-container-server },
{ name: swift_container_replicator, group: swift-container-server },
{ name: swift_container_updater, group: swift-container-server },
{ name: swift_rsyncd, group: swift-container-server },
{ name: swift_object_server, group: swift-object-server },
{ name: swift_object_auditor, group: swift-object-server },
{ name: swift_object_replication_server, group: swift-object-server },
{ name: swift_object_replicator, group: swift-object-server },
{ name: swift_object_updater, group: swift-object-server },
{ name: swift_object_expirer, group: swift-object-server },

View File

@ -75,6 +75,24 @@
- "/etc/localtime:/etc/localtime:ro"
when: inventory_hostname in groups['swift-account-server']
- name: Starting swift-account-replication-server container
become: true
kolla_docker:
action: "start_container"
common_options: "{{ docker_common_options }}"
environment:
RSYNC_CONNECT_PROG: "nc %H {{ swift_rsync_port }}"
image: "{{ swift_account_image_full }}"
name: "swift_account_replication_server"
volumes:
- "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
- "{{ node_config_directory }}/swift-account-replication-server/:{{ container_config_directory }}/:ro"
- "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}:shared"
- "/etc/localtime:/etc/localtime:ro"
when:
- inventory_hostname in groups['swift-account-server']
- swift_has_replication_network
- name: Starting swift-account-replicator container
become: true
kolla_docker:
@ -133,6 +151,24 @@
- "/etc/localtime:/etc/localtime:ro"
when: inventory_hostname in groups['swift-container-server']
- name: Starting swift-container-replication-server container
become: true
kolla_docker:
action: "start_container"
common_options: "{{ docker_common_options }}"
environment:
RSYNC_CONNECT_PROG: "nc %H {{ swift_rsync_port }}"
image: "{{ swift_container_image_full }}"
name: "swift_container_replication_server"
volumes:
- "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
- "{{ node_config_directory }}/swift-container-replication-server/:{{ container_config_directory }}/:ro"
- "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}:shared"
- "/etc/localtime:/etc/localtime:ro"
when:
- inventory_hostname in groups['swift-container-server']
- swift_has_replication_network
- name: Starting swift-container-replicator container
become: true
kolla_docker:
@ -191,6 +227,24 @@
- "/etc/localtime:/etc/localtime:ro"
when: inventory_hostname in groups['swift-object-server']
- name: Starting swift-object-replication-server container
become: true
kolla_docker:
action: "start_container"
common_options: "{{ docker_common_options }}"
environment:
RSYNC_CONNECT_PROG: "nc %H {{ swift_rsync_port }}"
image: "{{ swift_object_image_full }}"
name: "swift_object_replication_server"
volumes:
- "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
- "{{ node_config_directory }}/swift-object-replication-server/:{{ container_config_directory }}/:ro"
- "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}:shared"
- "/etc/localtime:/etc/localtime:ro"
when:
- inventory_hostname in groups['swift-object-server']
- swift_has_replication_network
- name: Starting swift-object-replicator container
become: true
kolla_docker:

View File

@ -1,5 +1,6 @@
{% set interface = swift_replication_interface if 'replicat' in service_name else swift_storage_interface %}
[DEFAULT]
bind_ip = {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}
bind_ip = {{ hostvars[inventory_hostname]['ansible_' + interface]['ipv4']['address'] }}
bind_port = {{ swift_account_server_port }}
devices = {{ swift_devices_mount_point }}
mount_check = false
@ -15,6 +16,9 @@ pipeline = account-server
[app:account-server]
use = egg:swift#account
{% if swift_has_replication_network %}
replication_server = {{ service_name == 'swift-account-replication-server' }}
{% endif %}
{% if service_name == 'swift-account-auditor' %}
[account-auditor]

View File

@ -1,5 +1,6 @@
{% set interface = swift_replication_interface if 'replicat' in service_name else swift_storage_interface %}
[DEFAULT]
bind_ip = {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}
bind_ip = {{ hostvars[inventory_hostname]['ansible_' + interface]['ipv4']['address'] }}
bind_port = {{ swift_container_server_port }}
devices = {{ swift_devices_mount_point }}
mount_check = false
@ -16,6 +17,9 @@ pipeline = container-server
[app:container-server]
use = egg:swift#container
allow_versions = True
{% if swift_has_replication_network %}
replication_server = {{ service_name == 'swift-container-replication-server' }}
{% endif %}
{% if service_name == 'swift-container-auditor' %}
[container-auditor]

View File

@ -1,5 +1,6 @@
{% set interface = swift_replication_interface if 'replicat' in service_name else swift_storage_interface %}
[DEFAULT]
bind_ip = {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}
bind_ip = {{ hostvars[inventory_hostname]['ansible_' + interface]['ipv4']['address'] }}
bind_port = {{ swift_object_server_port }}
devices = {{ swift_devices_mount_point }}
mount_check = false
@ -20,6 +21,9 @@ pipeline = object-server
[app:object-server]
use = egg:swift#object
{% if swift_has_replication_network %}
replication_server = {{ service_name == 'swift-object-replication-server' }}
{% endif %}
{% if service_name == 'swift-object-auditor' %}
[object-auditor]

View File

@ -1,4 +1,4 @@
address = {{ api_interface_address }}
address = {{ hostvars[inventory_hostname]['ansible_' + swift_replication_interface]['ipv4']['address'] }}
{% if inventory_hostname in groups['swift-account-server'] %}
[account]

View File

@ -0,0 +1,30 @@
{
"command": "swift-account-server /etc/swift/account-server.conf --verbose",
"config_files": [
{
"source": "/var/lib/kolla/swift/account.ring.gz",
"dest": "/etc/swift/account.ring.gz",
"owner": "swift",
"perm": "0640"
},
{
"source": "{{ container_config_directory }}/swift.conf",
"dest": "/etc/swift/swift.conf",
"owner": "swift",
"perm": "0640"
},
{
"source": "{{ container_config_directory }}/account-replication-server.conf",
"dest": "/etc/swift/account-server.conf",
"owner": "swift",
"perm": "0640"
},
{
"source": "{{ container_config_directory }}/policy.json",
"dest": "/etc/swift/policy.json",
"owner": "swift",
"perm": "0600",
"optional": true
}
]
}

View File

@ -0,0 +1,30 @@
{
"command": "swift-container-server /etc/swift/container-server.conf --verbose",
"config_files": [
{
"source": "/var/lib/kolla/swift/container.ring.gz",
"dest": "/etc/swift/container.ring.gz",
"owner": "swift",
"perm": "0640"
},
{
"source": "{{ container_config_directory }}/swift.conf",
"dest": "/etc/swift/swift.conf",
"owner": "swift",
"perm": "0640"
},
{
"source": "{{ container_config_directory }}/container-replication-server.conf",
"dest": "/etc/swift/container-server.conf",
"owner": "swift",
"perm": "0640"
},
{
"source": "{{ container_config_directory }}/policy.json",
"dest": "/etc/swift/policy.json",
"owner": "swift",
"perm": "0600",
"optional": true
}
]
}

View File

@ -0,0 +1,36 @@
{
"command": "swift-object-server /etc/swift/object-server.conf --verbose",
"config_files": [
{
"source": "/var/lib/kolla/swift/container.ring.gz",
"dest": "/etc/swift/container.ring.gz",
"owner": "swift",
"perm": "0640"
},
{
"source": "/var/lib/kolla/swift/object.ring.gz",
"dest": "/etc/swift/object.ring.gz",
"owner": "swift",
"perm": "0640"
},
{
"source": "{{ container_config_directory }}/swift.conf",
"dest": "/etc/swift/swift.conf",
"owner": "swift",
"perm": "0640"
},
{
"source": "{{ container_config_directory }}/object-replication-server.conf",
"dest": "/etc/swift/object-server.conf",
"owner": "swift",
"perm": "0640"
},
{
"source": "{{ container_config_directory }}/policy.json",
"dest": "/etc/swift/policy.json",
"owner": "swift",
"perm": "0600",
"optional": true
}
]
}

View File

@ -56,7 +56,7 @@ In Kolla operators should configure following network interfaces:
* ``storage_interface`` - This is the interface that is used by virtual
machines to communicate to Ceph. This can be heavily utilized so it's
recommended to put this network on 10Gig networking. Defaults to
recommended to use a high speed network fabric. Defaults to
``network_interface``.
* ``cluster_interface`` - This is another interface used by Ceph. It's used for
@ -64,6 +64,14 @@ In Kolla operators should configure following network interfaces:
bottleneck it can affect data consistency and performance of whole cluster.
Defaults to ``network_interface``.
* ``swift_storage_interface`` - This interface is used by Swift for storage
access traffic. This can be heavily utilized so it's recommended to use
a high speed network fabric. Defaults to ``storage_interface``.
* ``swift_replication_interface`` - This interface is used by Swift for storage
replication traffic. This can be heavily utilized so it's recommended to use
a high speed network fabric. Defaults to ``swift_storage_interface``.
* ``tunnel_interface`` - This interface is used by Neutron for vm-to-vm traffic
over tunneled networks (like VxLan). Defaults to ``network_interface``.

View File

@ -10,6 +10,24 @@ Overview
Kolla can deploy a full working Swift setup in either a **all-in-one** or
**multinode** setup.
Networking
~~~~~~~~~~
The following networks are used by Swift:
External API network (``kolla_external_vip_interface``)
This network is used by users to access the Swift public API.
Internal API network (``api_interface``)
This network is used by users to access the Swift internal API. It is also
used by HAProxy to access the Swift proxy servers.
Swift Storage network (``swift_storage_interface``)
This network is used by the Swift proxy server to access the account,
container and object servers. Defaults to ``storage_interface``.
Swift replication network (``swift_replication_network``)
This network is used for Swift storage replication traffic.
This is optional as the default configuration uses
the ``swift_storage_interface`` for replication traffic.
Disks with a partition table (recommended)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -75,6 +93,11 @@ generate rings for a demo setup. The commands work with **disks with partition
table** example listed above. Please modify accordingly if your setup is
different.
If using a separate replication network it is necessary to add the replication
network IP addresses to the rings. See the `Swift documentation
<https://docs.openstack.org/swift/latest/replication_network.html#dedicated-replication-network>`__
for details on how to do that.
Prepare for Rings generating
----------------------------
@ -162,6 +185,15 @@ To generate Swift container ring, run the following commands:
done
done
.. end
Rebalance
---------
To rebalance the ring files:
.. code-block:: console
for ring in object account container; do
docker run \
--rm \

View File

@ -90,6 +90,8 @@ kolla_internal_vip_address: "10.10.10.254"
#api_interface: "{{ network_interface }}"
#storage_interface: "{{ network_interface }}"
#cluster_interface: "{{ network_interface }}"
#swift_storage_interface: "{{ storage_interface }}"
#swift_replication_interface: "{{ swift_storage_interface }}"
#tunnel_interface: "{{ network_interface }}"
#dns_interface: "{{ network_interface }}"
#octavia_network_interface: "{{ api_interface }}"

View File

@ -0,0 +1,22 @@
---
features:
- |
Adds support to seperate Swift access and replication traffic from other
storage traffic.
In a deployment where both Ceph and Swift have been deployed,
this changes adds functionalality to support optional seperation
of storage network traffic. This adds two new network interfaces
'swift_storage_interface' and 'swift_replication_interface' which maintain
backwards compatibility.
The Swift access network interface is configured via 'swift_storage_interface',
which defaults to 'storage_interface'. The Swift replication network
interface is configured via 'swift_replication_interface', which
defaults to 'swift_storage_interface'.
If a separate replication network is used, Kolla Ansible now deploys separate
replication servers for the accounts, containers and objects, that listen on
this network. In this case, these services handle only replication traffic, and
the original account-, container- and object- servers only handle storage
user requests.