From 486fb96a32a75d84621b693709df8b60d7d52019 Mon Sep 17 00:00:00 2001 From: Alena Holanova Date: Thu, 4 Feb 2016 16:29:06 +0100 Subject: [PATCH] initial formula - add configs,metadata,readme --- .gitignore | 4 + README.md => README.rst | 41 ++++-- debian/changelog | 5 + debian/compat | 1 + debian/control | 15 +++ debian/copyright | 15 +++ debian/docs | 3 + debian/install | 2 + debian/rules | 5 + debian/source/format | 1 + metadata.yml | 3 + metadata/service/proxy/cluster.yml | 35 +++++ metadata/service/proxy/single.yml | 31 +++++ metadata/service/server/cluster.yml | 50 ------- metadata/service/storage/cluster.yml | 27 ++++ metadata/service/storage/single.yml | 27 ++++ swift/account.sls | 11 +- swift/common.sls | 12 +- swift/conf/account-server.conf | 26 ---- swift/conf/container-server.conf | 28 ---- swift/conf/object-server.conf | 26 ---- swift/conf/proxy-server.conf | 64 --------- swift/conf/swift.conf | 3 - swift/container.sls | 18 +-- swift/files/kilo/account-server.conf | 32 +++++ swift/files/kilo/container-server.conf | 35 +++++ swift/files/kilo/object-server.conf | 35 +++++ swift/files/kilo/proxy-server.conf | 100 ++++++++++++++ swift/files/kilo/swift.conf | 174 +++++++++++++++++++++++++ swift/init.sls | 3 + map.jinja => swift/map.jinja | 9 ++ swift/object.sls | 6 - swift/proxy.sls | 11 +- tests/pillar/proxy_cluster.sls | 32 +++++ tests/run_tests.sh | 160 +++++++++++++++++++++++ 35 files changed, 799 insertions(+), 251 deletions(-) create mode 100644 .gitignore rename README.md => README.rst (64%) create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/docs create mode 100644 debian/install create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100644 metadata.yml create mode 100644 metadata/service/proxy/cluster.yml create mode 100644 metadata/service/proxy/single.yml delete mode 100644 metadata/service/server/cluster.yml create mode 100644 metadata/service/storage/cluster.yml create mode 100644 metadata/service/storage/single.yml delete mode 100644 swift/conf/account-server.conf delete mode 100644 swift/conf/container-server.conf delete mode 100644 swift/conf/object-server.conf delete mode 100644 swift/conf/proxy-server.conf delete mode 100644 swift/conf/swift.conf create mode 100644 swift/files/kilo/account-server.conf create mode 100644 swift/files/kilo/container-server.conf create mode 100644 swift/files/kilo/object-server.conf create mode 100644 swift/files/kilo/proxy-server.conf create mode 100644 swift/files/kilo/swift.conf rename map.jinja => swift/map.jinja (91%) create mode 100644 tests/pillar/proxy_cluster.sls create mode 100755 tests/run_tests.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1bfce6e --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +tests/build/ +*.swp +*.pyc +.ropeproject \ No newline at end of file diff --git a/README.md b/README.rst similarity index 64% rename from README.md rename to README.rst index 552a9bb..38536ff 100644 --- a/README.md +++ b/README.rst @@ -1,16 +1,26 @@ - -# Swift +================ +OpenStack Swift +================ Swift is a highly available, distributed, eventually consistent object/blob store. Organizations can use Swift to store lots of data efficiently, safely, and cheaply. -## Sample pillars +Sample pillars +============== -### Swift proxy server +Swift proxy server +------------------ + +.. code-block:: yaml swift: - proxy: + common: + enabled: true + version: kilo + swift_hash_path_suffix: hash + swift_hash_path_prefix: hash + proxy: + version: kilo enabled: true - secret_key: shared_hash_suffix bind: address: 0.0.0.0 port: 8080 @@ -22,29 +32,38 @@ Swift is a highly available, distributed, eventually consistent object/blob stor password: pwd tenant: service -### Swift storage server +Swift storage server +-------------------- + +.. code-block:: yaml swift: + common: + version: kilo + enabled: true + swift_hash_path_suffix: hash + swift_hash_path_prefix: hash object: enabled: true - secret_key: shared_hash_suffix + version: kilo bind: address: 0.0.0.0 port: 6000 container: enabled: true - secret_key: shared_hash_suffix + version: kilo bind: address: 0.0.0.0 port: 6001 account: enabled: true - secret_key: shared_hash_suffix + version: kilo bind: address: 0.0.0.0 port: 6002 -## Read more +Read more +========= * http://docs.openstack.org/developer/swift/overview_architecture.html * http://docs.openstack.org/developer/swift/howto_installmultinode.html diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..69001c7 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +salt-formula-swift (0.1) trusty; urgency=medium + + * Initial release + + -- Jakub Pavlik Thu, 13 Aug 2015 23:23:41 +0200 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..ca40546 --- /dev/null +++ b/debian/control @@ -0,0 +1,15 @@ +Source: salt-formula-swift +Maintainer: Jakub Pavlik +Section: admin +Priority: optional +Build-Depends: debhelper (>= 9) +Standards-Version: 3.9.6 +Homepage: http://www.tcpcloud.eu +Vcs-Browser: https://github.com/tcpcloud/salt-formula-swift +Vcs-Git: https://github.com/tcpcloud/salt-formula-swift.git + +Package: salt-formula-swift +Architecture: all +Depends: ${misc:Depends}, salt-master, reclass +Description: Nova Salt formula + Install and configure Nova server and client. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..32faa67 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,15 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: salt-formula-swift +Upstream-Contact: Jakub Pavlik +Source: https://github.com/tcpcloud/salt-formula-swift + +Files: * +Copyright: 2014-2015 tcp cloud +License: Apache-2.0 + Copyright (C) 2014-2015 tcp cloud + . + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + . + On a Debian system you can find a copy of this license in + /usr/share/common-licenses/Apache-2.0. diff --git a/debian/docs b/debian/docs new file mode 100644 index 0000000..d585829 --- /dev/null +++ b/debian/docs @@ -0,0 +1,3 @@ +README.rst +CHANGELOG.rst +VERSION diff --git a/debian/install b/debian/install new file mode 100644 index 0000000..f8f6739 --- /dev/null +++ b/debian/install @@ -0,0 +1,2 @@ +swift/* /usr/share/salt-formulas/env/swift/ +metadata/service/* /usr/share/salt-formulas/reclass/service/swift/ diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..abde6ef --- /dev/null +++ b/debian/rules @@ -0,0 +1,5 @@ +#!/usr/bin/make -f + +%: + dh $@ + diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/metadata.yml b/metadata.yml new file mode 100644 index 0000000..b9d2d6c --- /dev/null +++ b/metadata.yml @@ -0,0 +1,3 @@ +name: "swift" +version: "0.2" +source: "https://github.com/github/salt-formula-swift" diff --git a/metadata/service/proxy/cluster.yml b/metadata/service/proxy/cluster.yml new file mode 100644 index 0000000..36ca5fa --- /dev/null +++ b/metadata/service/proxy/cluster.yml @@ -0,0 +1,35 @@ +applications: +- swift +parameters: + swift: + common: + enabled: true + version: ${_param:swift_version} + swift_hash_path_suffix: ${_param:swift_swift_hash_path_suffix} + swift_hash_path_prefix: ${_param:swift_swift_hash_path_prefix} + proxy: + enabled: true + version: ${_param:swift_version} + bind: + address: 0.0.0.0 + port: 8080 + cache: + engine: memcached + members: + - host: ${_param:cluster_node01_address} + port: 11211 + - host: ${_param:cluster_node02_address} + port: 11211 + - host: ${_param:cluster_node03_address} + port: 11211 + ring: + partition_power: 3 + replicas: 3 + hours: 1 + identity: + engine: keystone + host: ${_param:cluster_vip_address} + port: 35357 + user: swift + password: ${_param:keystone_swift_password} + tenant: service \ No newline at end of file diff --git a/metadata/service/proxy/single.yml b/metadata/service/proxy/single.yml new file mode 100644 index 0000000..166f1fb --- /dev/null +++ b/metadata/service/proxy/single.yml @@ -0,0 +1,31 @@ +applications: +- swift +parameters: + swift: + common: + enabled: true + version: ${_param:swift_version} + swift_hash_path_suffix: ${_param:swift_swift_hash_path_suffix} + swift_hash_path_prefix: ${_param:swift_swift_hash_path_prefix} + proxy: + enabled: true + version: ${_param:swift_version} + bind: + address: 0.0.0.0 + port: 8080 + cache: + engine: memcached + members: + - host: 127.0.0.1 + port: 11211 + ring: + partition_power: 3 + replicas: 3 + hours: 1 + identity: + engine: keystone + host: ${_param:single_address} + port: 35357 + user: swift + password: ${_param:keystone_swift_password} + tenant: service \ No newline at end of file diff --git a/metadata/service/server/cluster.yml b/metadata/service/server/cluster.yml deleted file mode 100644 index eb0586d..0000000 --- a/metadata/service/server/cluster.yml +++ /dev/null @@ -1,50 +0,0 @@ -applications: -- swift -parameters: - swift: - proxy: - enabled: true - secret_key: shared_hash_suffix - bind: - address: ${linux:network:host:local:address} - port: 8080 - cache: - members: - - host: ${linux:network:host:ctl01:address} - port: 11211 - - host: ${linux:network:host:ctl02:address} - port: 11211 - - host: ${linux:network:host:ctl03:address} - port: 11211 - ring: - partition_power: 3 - replicas: 3 - hours: 1 - identity: - engine: keystone - host: ${linux:network:host:vip:address} - port: 35357 - user: swift - password: vuugri3ubrisebfsuir23 - tenant: service - haproxy: - proxy: - listens: - - name: swift - type: general-service - binds: - - address: ${linux:network:host:vip:address} - port: 8080 - servers: - - name: ctl01 - host: ${linux:network:host:ctl01:address} - port: 8080 - params: check - - name: ctl02 - host: ${linux:network:host:ctl02:address} - port: 8080 - params: check - - name: ctl03 - host: ${linux:network:host:ctl03:address} - port: 8080 - params: check diff --git a/metadata/service/storage/cluster.yml b/metadata/service/storage/cluster.yml new file mode 100644 index 0000000..34c2256 --- /dev/null +++ b/metadata/service/storage/cluster.yml @@ -0,0 +1,27 @@ +applications: +- swift +parameters: + swift: + common: + enabled: true + version: ${_param:swift_version} + swift_hash_path_suffix: ${_param:swift_swift_hash_path_suffix} + swift_hash_path_prefix: ${_param:swift_swift_hash_path_prefix} + object: + enabled: true + version: ${_param:swift_version} + bind: + address: 0.0.0.0 + port: 6000 + container: + enabled: true + version: ${_param:swift_version} + bind: + address: 0.0.0.0 + port: 6001 + account: + enabled: true + version: ${_param:swift_version} + bind: + address: 0.0.0.0 + port: 6002 \ No newline at end of file diff --git a/metadata/service/storage/single.yml b/metadata/service/storage/single.yml new file mode 100644 index 0000000..34c2256 --- /dev/null +++ b/metadata/service/storage/single.yml @@ -0,0 +1,27 @@ +applications: +- swift +parameters: + swift: + common: + enabled: true + version: ${_param:swift_version} + swift_hash_path_suffix: ${_param:swift_swift_hash_path_suffix} + swift_hash_path_prefix: ${_param:swift_swift_hash_path_prefix} + object: + enabled: true + version: ${_param:swift_version} + bind: + address: 0.0.0.0 + port: 6000 + container: + enabled: true + version: ${_param:swift_version} + bind: + address: 0.0.0.0 + port: 6001 + account: + enabled: true + version: ${_param:swift_version} + bind: + address: 0.0.0.0 + port: 6002 \ No newline at end of file diff --git a/swift/account.sls b/swift/account.sls index a010550..eb8ed41 100644 --- a/swift/account.sls +++ b/swift/account.sls @@ -2,19 +2,13 @@ {%- if account.enabled %} -include: -- swift.common - swift_account_packages: pkg.installed: - names: {{ account.pkgs }} - - require_in: - - file: swift_config -swift_account_config: +/etc/swift/account-server.conf: file.managed: - - name: /etc/swift/account-server.conf - - source: salt://swift/conf/account-server.conf + - source: salt://swift/files/{{ proxy.version }}/account-server.conf - template: jinja - user: swift - group: swift @@ -24,7 +18,6 @@ swift_account_services: service.running: - names: {{ account.services }} - watch: - - file: swift_config - file: swift_account_config {%- endif %} diff --git a/swift/common.sls b/swift/common.sls index 171832f..7656e1c 100644 --- a/swift/common.sls +++ b/swift/common.sls @@ -1,8 +1,14 @@ +{% from "swift/map.jinja" import common with context %} -: +swift_common_packages: + pkg.installed: + - names: {{ common.pkgs }} + - require_in: + - file: /etc/swift/swift.conf + +/etc/swift/swift.conf: file.managed: - - name: /etc/swift/swift.conf - - source: salt://swift/conf/swift.conf + - source: salt://swift/files/{{ common.version }}/swift.conf - template: jinja - user: swift - group: swift diff --git a/swift/conf/account-server.conf b/swift/conf/account-server.conf deleted file mode 100644 index e8e29ee..0000000 --- a/swift/conf/account-server.conf +++ /dev/null @@ -1,26 +0,0 @@ -{% from "swift/map.jinja" import account with context %} - -[DEFAULT] -devices = /var/lib/glance/node -bind_ip = 192.168.1.2 -bind_port = 6002 -mount_check = false -user = swift -log_facility = LOG_SYSLOG -log_level = INFO -log_name = swift-account-server -workers = 1 - -[pipeline:main] -pipeline = account-server - -[app:account-server] -use = egg:swift#account - -[account-replicator] -concurrency = 8 - -[account-auditor] - -[account-reaper] -concurrency = 8 \ No newline at end of file diff --git a/swift/conf/container-server.conf b/swift/conf/container-server.conf deleted file mode 100644 index 79a591c..0000000 --- a/swift/conf/container-server.conf +++ /dev/null @@ -1,28 +0,0 @@ -{% from "swift/map.jinja" import container with context %} - -[DEFAULT] -devices = /var/lib/glance/node -bind_ip = 192.168.1.2 -bind_port = 6001 -mount_check = false -user = swift -log_facility = LOG_SYSLOG -log_level = INFO -log_name = swift-container-server -workers = 1 - -[pipeline:main] -pipeline = container-server - -[app:container-server] -use = egg:swift#container - -[container-replicator] -concurrency = 8 - -[container-updater] -concurrency = 8 - -[container-auditor] - -[container-sync] \ No newline at end of file diff --git a/swift/conf/object-server.conf b/swift/conf/object-server.conf deleted file mode 100644 index 6dbdc66..0000000 --- a/swift/conf/object-server.conf +++ /dev/null @@ -1,26 +0,0 @@ -{% from "swift/map.jinja" import object with context %} - -[DEFAULT] -devices = /var/lib/glance/node -bind_ip = {{ object.bind.address }} -bind_port = {{ object.bind.port }} 6000 -mount_check = false -user = swift -log_facility = LOG_SYSLOG -log_level = INFO -log_name = swift-object-server -workers = 1 - -[pipeline:main] -pipeline = object-server - -[app:object-server] -use = egg:swift#object - -[object-replicator] -concurrency = 8 - -[object-updater] -concurrency = 8 - -[object-auditor] \ No newline at end of file diff --git a/swift/conf/proxy-server.conf b/swift/conf/proxy-server.conf deleted file mode 100644 index 26b4c6e..0000000 --- a/swift/conf/proxy-server.conf +++ /dev/null @@ -1,64 +0,0 @@ -{% from "swift/map.jinja" import proxy with context %} - -[DEFAULT] -bind_port = {{ proxy.bind.port }} -bind_ip = <%= @proxy_local_net_ip %> -user = swift -workers = {{ proxy.workers }} - -#log_name = swift -#log_facility = <%= @log_facility %> -#log_level = <%= @log_level %> -#log_headers = <%= @log_headers %> -#log_address = <%= @log_address %> -#<% if @log_udp_host != '' -%> -# If set, log_udp_host will override log_address -#log_udp_host = <%= @log_udp_host -%> -#<% end %> -#<% if @log_udp_host !='' and @log_udp_port != '' -%> -#log_udp_port = <%= @log_udp_port -%> - -[pipeline:main] -pipeline = healthcheck cache {% if proxy.identity is defined %}authtoken keystoneauth {% endif %}proxy-server -#pipeline = <%= @pipeline.to_a.join(' ') %> - -[app:proxy-server] -use = egg:swift#proxy -allow_account_management = true -account_autocreate = true -#set log_name = proxy-server -#set log_facility = <%= @log_facility %> -#set log_level = <%= @log_level %> -#set log_address = <%= @log_address %> -#log_handoffs = <%= @log_handoffs %> - -{%- if proxy.identity is defined %} -[filter:keystoneauth] -use = egg:swift#keystoneauth -operator_roles = Member,admin,swiftoperator - -[filter:authtoken] -paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory -# Delaying the auth decision is required to support token-less -# usage for anonymous referrers ('.r:*'). -delay_auth_decision = true -# cache directory for signing certificate -signing_dir = /home/swift/keystone-signing -# auth_* settings refer to the Keystone server -auth_protocol = http -auth_host = {{ proxy.identity.host }} -auth_port = {{ proxy.identity.port }} -# the service tenant and swift username and password created in Keystone -admin_tenant_name = {{ proxy.identity.tenant }} -admin_user = {{ proxy.identity.host }} -admin_password = {{ proxy.identity.password }} -{%- endif %} - -[filter:cache] -use = egg:swift#memcache - -[filter:catch_errors] -use = egg:swift#catch_errors - -[filter:healthcheck] -use = egg:swift#healthcheck diff --git a/swift/conf/swift.conf b/swift/conf/swift.conf deleted file mode 100644 index 32673f6..0000000 --- a/swift/conf/swift.conf +++ /dev/null @@ -1,3 +0,0 @@ - -[swift-hash] -swift_hash_path_suffix = {% if pillar.swift.proxy is defined %}{{ pillar.swift.proxy.secret_key }}{% else %}{{ pillar.swift.container.secret_key }}{% endif %} diff --git a/swift/container.sls b/swift/container.sls index 8da0e50..023e77f 100644 --- a/swift/container.sls +++ b/swift/container.sls @@ -2,26 +2,13 @@ {%- if container.enabled %} -include: -- swift.common - -/var/swift/recon: - file.directory: - - user: swift - - group: swift - - require: - - pkg: swift_container_packages - swift_container_packages: pkg.installed: - names: {{ container.pkgs }} - - require_in: - - file: swift_config -swift_container_config: +/etc/swift/container-server.conf: file.managed: - - name: /etc/swift/container-server.conf - - source: salt://swift/conf/container-server.conf + - source: salt://swift/files/{{ proxy.version }}/container-server.conf - template: jinja - user: swift - group: swift @@ -31,7 +18,6 @@ swift_container_services: service.running: - names: {{ container.services }} - watch: - - file: swift_config - file: swift_container_config {%- endif %} diff --git a/swift/files/kilo/account-server.conf b/swift/files/kilo/account-server.conf new file mode 100644 index 0000000..c7d4e47 --- /dev/null +++ b/swift/files/kilo/account-server.conf @@ -0,0 +1,32 @@ +{% from "swift/map.jinja" import account with context %} + +[DEFAULT] +bind_ip = {{ account.bind.address }} +bind_port = {{ account.bind.port }} +# bind_timeout = 30 +# backlog = 4096 +user = swift +swift_dir = /etc/swift +devices = /srv/node + +[pipeline:main] +pipeline = healthcheck recon account-server + +[app:account-server] +use = egg:swift#account + +[filter:healthcheck] +use = egg:swift#healthcheck + +[filter:recon] +use = egg:swift#recon +recon_cache_path = /var/cache/swift + +[account-replicator] + +[account-auditor] + +[account-reaper] + +[filter:xprofile] +use = egg:swift#xprofile diff --git a/swift/files/kilo/container-server.conf b/swift/files/kilo/container-server.conf new file mode 100644 index 0000000..3d9a48e --- /dev/null +++ b/swift/files/kilo/container-server.conf @@ -0,0 +1,35 @@ +{% from "swift/map.jinja" import container with context %} + +[DEFAULT] +bind_ip = {{ container.bind.address }} +bind_port = {{ container.bind.port }} +# bind_timeout = 30 +# backlog = 4096 +user = swift +swift_dir = /etc/swift +devices = /srv/node + +[pipeline:main] +pipeline = healthcheck recon container-server + +[app:container-server] +use = egg:swift#container + +[filter:healthcheck] +use = egg:swift#healthcheck + +[filter:recon] +use = egg:swift#recon +recon_cache_path = /var/cache/swift + +[container-replicator] + +[container-updater] + +[container-auditor] + +[container-sync] + +[filter:xprofile] +use = egg:swift#xprofile + diff --git a/swift/files/kilo/object-server.conf b/swift/files/kilo/object-server.conf new file mode 100644 index 0000000..be7c5e4 --- /dev/null +++ b/swift/files/kilo/object-server.conf @@ -0,0 +1,35 @@ +{% from "swift/map.jinja" import object with context %} + +[DEFAULT] +bind_ip = {{ object.bind.address }} +bind_port = {{ object.bind.port }} +# bind_timeout = 30 +# backlog = 4096 +user = swift +swift_dir = /etc/swift +devices = /srv/node + +[pipeline:main] +pipeline = healthcheck recon object-server + +[app:object-server] +use = egg:swift#object + +[filter:healthcheck] +use = egg:swift#healthcheck + +[filter:recon] +use = egg:swift#recon +recon_cache_path = /var/cache/swift +recon_lock_path = /var/lock + +[object-replicator] + +[object-reconstructor] + +[object-updater] + +[object-auditor] + +[filter:xprofile] +use = egg:swift#xprofile \ No newline at end of file diff --git a/swift/files/kilo/proxy-server.conf b/swift/files/kilo/proxy-server.conf new file mode 100644 index 0000000..1d08a96 --- /dev/null +++ b/swift/files/kilo/proxy-server.conf @@ -0,0 +1,100 @@ +{% from "swift/map.jinja" import proxy with context %} + +[DEFAULT] +bind_ip = {{ proxy.bind.address }} +bind_port = {{ proxy.bind.port }} +swift_dir = /etc/swift +user = swift + +workers = {{ proxy.workers }} +log_level = DEBUG + +[pipeline:main] +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk ratelimit {% if proxy.identity is defined %}authtoken keystoneauth {% endif %} formpost staticweb container-quotas account-quotas slo dlo proxy-logging proxy-server + +[app:proxy-server] +account_autocreate = true +conn_timeout = 20 +node_timeout = 120 +use = egg:swift#proxy + +#[filter:tempauth] +#use = egg:swift#tempauth +#reseller_prefix = TEMPAUTH + +#user_admin_admin = admin .admin .reseller_admin +#user_test_tester = testing .admin +#user_test2_tester2 = testing2 .admin +#user_test_tester3 = testing3 +#user_test5_tester5 = testing5 service + +[filter:crossdomain] +use = egg:swift#crossdomain + +{%- if proxy.identity is defined %} +[filter:authtoken] +paste.filter_factory = keystonemiddleware.auth_token:filter_factory +auth_url = http://{{ proxy.identity.host }}:35357/ +auth_uri = http://{{ proxy.identity.host }}:5000/ +tenant_name = {{ proxy.identity.tenant }} +username = {{ proxy.identity.username }} +password = {{ proxy.identity.password }} +delay_auth_decision = true +auth_plugin = password + +signing_dir = /var/cache/swift +cache = swift.cache +include_service_catalog = False + +[filter:keystoneauth] +use = egg:swift#keystoneauth +operator_roles = admin, Member +{%- endif %} + +[filter:healthcheck] +use = egg:swift#healthcheck + +[filter:cache] +use = egg:swift#memcache +memcached_servers={%- for member in proxy.cache.members %}{{ member.host }}:{{ member.port }}{% if not loop.last %},{% endif %}{%- endfor %} + +[filter:ratelimit] +use = egg:swift#ratelimit + +[filter:catch_errors] +use = egg:swift#catch_errors + +[filter:staticweb] +use = egg:swift#staticweb + +[filter:tempurl] +use = egg:swift#tempurl + +[filter:formpost] +use = egg:swift#formpost + +[filter:proxy-logging] +reveal_sensitive_prefix = 12 +use = egg:swift#proxy_logging + +[filter:bulk] +use = egg:swift#bulk + +[filter:slo] +use = egg:swift#slo + +[filter:dlo] +use = egg:swift#dlo + +[filter:container-quotas] +use = egg:swift#container_quotas + +[filter:account-quotas] +use = egg:swift#account_quotas + +[filter:gatekeeper] + +use = egg:swift#gatekeeper + +[filter:container_sync] +use = egg:swift#container_sync diff --git a/swift/files/kilo/swift.conf b/swift/files/kilo/swift.conf new file mode 100644 index 0000000..da6bd09 --- /dev/null +++ b/swift/files/kilo/swift.conf @@ -0,0 +1,174 @@ +[swift-hash] + +# swift_hash_path_suffix and swift_hash_path_prefix are used as part of the +# the hashing algorithm when determining data placement in the cluster. +# These values should remain secret and MUST NOT change +# once a cluster has been deployed. + +swift_hash_path_suffix = {{ pillar.swift.common:swift_hash_path_suffix }} +swift_hash_path_prefix = {{ pillar.swift.common:swift_hash_path_prefix }} + +# storage policies are defined here and determine various characteristics +# about how objects are stored and treated. Policies are specified by name on +# a per container basis. Names are case-insensitive. The policy index is +# specified in the section header and is used internally. The policy with +# index 0 is always used for legacy containers and can be given a name for use +# in metadata however the ring file name will always be 'object.ring.gz' for +# backwards compatibility. If no policies are defined a policy with index 0 +# will be automatically created for backwards compatibility and given the name +# Policy-0. A default policy is used when creating new containers when no +# policy is specified in the request. If no other policies are defined the +# policy with index 0 will be declared the default. If multiple policies are +# defined you must define a policy with index 0 and you must specify a +# default. It is recommended you always define a section for +# storage-policy:0. +# +# A 'policy_type' argument is also supported but is not mandatory. Default +# policy type 'replication' is used when 'policy_type' is unspecified. +[storage-policy:0] +name = Policy-0 +default = yes +#policy_type = replication + +# the following section would declare a policy called 'silver', the number of +# replicas will be determined by how the ring is built. In this example the +# 'silver' policy could have a lower or higher # of replicas than the +# 'Policy-0' policy above. The ring filename will be 'object-1.ring.gz'. You +# may only specify one storage policy section as the default. If you changed +# this section to specify 'silver' as the default, when a client created a new +# container w/o a policy specified, it will get the 'silver' policy because +# this config has specified it as the default. However if a legacy container +# (one created with a pre-policy version of swift) is accessed, it is known +# implicitly to be assigned to the policy with index 0 as opposed to the +# current default. +#[storage-policy:1] +#name = silver +#policy_type = replication + +# The following declares a storage policy of type 'erasure_coding' which uses +# Erasure Coding for data reliability. The 'erasure_coding' storage policy in +# Swift is available as a "beta". Please refer to Swift documentation for +# details on how the 'erasure_coding' storage policy is implemented. +# +# Swift uses PyECLib, a Python Erasure coding API library, for encode/decode +# operations. Please refer to Swift documentation for details on how to +# install PyECLib. +# +# When defining an EC policy, 'policy_type' needs to be 'erasure_coding' and +# EC configuration parameters 'ec_type', 'ec_num_data_fragments' and +# 'ec_num_parity_fragments' must be specified. 'ec_type' is chosen from the +# list of EC backends supported by PyECLib. The ring configured for the +# storage policy must have it's "replica" count configured to +# 'ec_num_data_fragments' + 'ec_num_parity_fragments' - this requirement is +# validated when services start. 'ec_object_segment_size' is the amount of +# data that will be buffered up before feeding a segment into the +# encoder/decoder. More information about these configuration options and +# supported `ec_type` schemes is available in the Swift documentation. Please +# refer to Swift documentation for details on how to configure EC policies. +# +# The example 'deepfreeze10-4' policy defined below is a _sample_ +# configuration with 10 'data' and 4 'parity' fragments. 'ec_type' +# defines the Erasure Coding scheme. 'jerasure_rs_vand' (Reed-Solomon +# Vandermonde) is used as an example below. +#[storage-policy:2] +#name = deepfreeze10-4 +#policy_type = erasure_coding +#ec_type = jerasure_rs_vand +#ec_num_data_fragments = 10 +#ec_num_parity_fragments = 4 +#ec_object_segment_size = 1048576 + + +# The swift-constraints section sets the basic constraints on data +# saved in the swift cluster. These constraints are automatically +# published by the proxy server in responses to /info requests. + +[swift-constraints] + +# max_file_size is the largest "normal" object that can be saved in +# the cluster. This is also the limit on the size of each segment of +# a "large" object when using the large object manifest support. +# This value is set in bytes. Setting it to lower than 1MiB will cause +# some tests to fail. It is STRONGLY recommended to leave this value at +# the default (5 * 2**30 + 2). + +#max_file_size = 5368709122 + + +# max_meta_name_length is the max number of bytes in the utf8 encoding +# of the name portion of a metadata header. + +#max_meta_name_length = 128 + + +# max_meta_value_length is the max number of bytes in the utf8 encoding +# of a metadata value + +#max_meta_value_length = 256 + + +# max_meta_count is the max number of metadata keys that can be stored +# on a single account, container, or object + +#max_meta_count = 90 + + +# max_meta_overall_size is the max number of bytes in the utf8 encoding +# of the metadata (keys + values) + +#max_meta_overall_size = 4096 +# max_header_size is the max number of bytes in the utf8 encoding of each +# header. Using 8192 as default because eventlet use 8192 as max size of +# header line. This value may need to be increased when using identity +# v3 API tokens including more than 7 catalog entries. +# See also include_service_catalog in proxy-server.conf-sample +# (documented in overview_auth.rst) + +#max_header_size = 8192 + + +# By default the maximum number of allowed headers depends on the number of max +# allowed metadata settings plus a default value of 32 for regular http +# headers. If for some reason this is not enough (custom middleware for +# example) it can be increased with the extra_header_count constraint. + +#extra_header_count = 32 + + +# max_object_name_length is the max number of bytes in the utf8 encoding +# of an object name + +#max_object_name_length = 1024 + + +# container_listing_limit is the default (and max) number of items +# returned for a container listing request + +#container_listing_limit = 10000 + + +# account_listing_limit is the default (and max) number of items returned +# for an account listing request +#account_listing_limit = 10000 + + +# max_account_name_length is the max number of bytes in the utf8 encoding +# of an account name + +#max_account_name_length = 256 + + +# max_container_name_length is the max number of bytes in the utf8 encoding +# of a container name + +#max_container_name_length = 256 + +# By default all REST API calls should use "v1" or "v1.0" as the version string, +# for example "/v1/account". This can be manually overridden to make this +# backward-compatible, in case a different version string has been used before. +# Use a comma-separated list in case of multiple allowed versions, for example +# valid_api_versions = v0,v1,v2 +# This is only enforced for account, container and object requests. The allowed +# api versions are by default excluded from /info. + +# valid_api_versions = v1,v1.0 \ No newline at end of file diff --git a/swift/init.sls b/swift/init.sls index 75bad65..177ef10 100644 --- a/swift/init.sls +++ b/swift/init.sls @@ -1,5 +1,8 @@ include: +{% if pillar.swift.common is defined %} +- swift.common +{% endif %} {%- if pillar.swift.object is defined %} - swift.object {%- endif %} diff --git a/map.jinja b/swift/map.jinja similarity index 91% rename from map.jinja rename to swift/map.jinja index 1296060..c2ed627 100644 --- a/map.jinja +++ b/swift/map.jinja @@ -1,4 +1,13 @@ +{% set common = salt['grains.filter_by']({ + 'Debian': { + 'pkgs': ['swift'] + }, + 'RedHat': { + 'pkgs': ['openstack-swift'] + }, +}, merge=salt['pillar.get']('swift:common')) %} + {% set proxy = salt['grains.filter_by']({ 'Debian': { 'pkgs': ['swift', 'python-swiftclient', 'swift-proxy'], diff --git a/swift/object.sls b/swift/object.sls index 07ce56b..89f2cdb 100644 --- a/swift/object.sls +++ b/swift/object.sls @@ -2,14 +2,9 @@ {%- if object.enabled %} -include: -- swift.common - swift_object_packages: pkg.installed: - names: {{ object.pkgs }} - - require_in: - - file: swift_config swift_object_config: file.managed: @@ -24,7 +19,6 @@ swift_object_services: service.running: - names: {{ object.services }} - watch: - - file: swift_config - file: swift_object_config {%- endif %} diff --git a/swift/proxy.sls b/swift/proxy.sls index fcf8a0a..2573f6c 100644 --- a/swift/proxy.sls +++ b/swift/proxy.sls @@ -2,19 +2,13 @@ {%- if proxy.enabled %} -include: -- swift.common - swift_proxy_packages: pkg.installed: - names: {{ proxy.pkgs }} - - require_in: - - file: swift_proxy_config -swift_proxy_config: +/etc/swift/proxy-server.conf: file.managed: - - name: /etc/swift/proxy-server.conf - - source: salt://swift/conf/proxy-server.conf + - source: salt://swift/files/{{ proxy.version }}/proxy-server.conf - template: jinja - user: swift - group: swift @@ -25,6 +19,5 @@ swift_proxy_services: - names: {{ proxy.services }} - watch: - file: swift_proxy_config - - file: swift_config {%- endif %} diff --git a/tests/pillar/proxy_cluster.sls b/tests/pillar/proxy_cluster.sls new file mode 100644 index 0000000..8d946d4 --- /dev/null +++ b/tests/pillar/proxy_cluster.sls @@ -0,0 +1,32 @@ +swift: + common: + enabled: true + version: kilo + swift_hash_path_suffix: myhash + swift_hash_path_prefix: myhash + proxy: + enabled: true + version: kilo + bind: + address: 0.0.0.0 + port: 8080 + cache: + engine: memcached + members: + - host: 127.0.0.1 + port: 11211 + - host: 127.0.0.1 + port: 11211 + - host: 127.0.0.1 + port: 11211 + ring: + partition_power: 3 + replicas: 3 + hours: 1 + identity: + engine: keystone + host: 127.0.0.1 + port: 35357 + user: swift + password: password + tenant: service \ No newline at end of file diff --git a/tests/run_tests.sh b/tests/run_tests.sh new file mode 100755 index 0000000..1f1316f --- /dev/null +++ b/tests/run_tests.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +set -e +[ -n "$DEBUG" ] && set -x + +CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +METADATA=${CURDIR}/../metadata.yml +FORMULA_NAME=$(cat $METADATA | python -c "import sys,yaml; print yaml.load(sys.stdin)['name']") + +## Overrideable parameters +PILLARDIR=${PILLARDIR:-${CURDIR}/pillar} +BUILDDIR=${BUILDDIR:-${CURDIR}/build} +VENV_DIR=${VENV_DIR:-${BUILDDIR}/virtualenv} +DEPSDIR=${BUILDDIR}/deps + +SALT_FILE_DIR=${SALT_FILE_DIR:-${BUILDDIR}/file_root} +SALT_PILLAR_DIR=${SALT_PILLAR_DIR:-${BUILDDIR}/pillar_root} +SALT_CONFIG_DIR=${SALT_CONFIG_DIR:-${BUILDDIR}/salt} +SALT_CACHE_DIR=${SALT_CACHE_DIR:-${SALT_CONFIG_DIR}/cache} + +SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR}" + +if [ "x${SALT_VERSION}" != "x" ]; then + PIP_SALT_VERSION="==${SALT_VERSION}" +fi + +## Functions +log_info() { + echo "[INFO] $*" +} + +log_err() { + echo "[ERROR] $*" >&2 +} + +setup_virtualenv() { + log_info "Setting up Python virtualenv" + virtualenv $VENV_DIR + source ${VENV_DIR}/bin/activate + pip install salt${PIP_SALT_VERSION} +} + +setup_pillar() { + [ ! -d ${SALT_PILLAR_DIR} ] && mkdir -p ${SALT_PILLAR_DIR} + echo "base:" > ${SALT_PILLAR_DIR}/top.sls + for pillar in ${PILLARDIR}/*; do + state_name=$(basename ${pillar%.sls}) + echo -e " ${state_name}:\n - ${state_name}" >> ${SALT_PILLAR_DIR}/top.sls + done +} + +setup_salt() { + [ ! -d ${SALT_FILE_DIR} ] && mkdir -p ${SALT_FILE_DIR} + [ ! -d ${SALT_CONFIG_DIR} ] && mkdir -p ${SALT_CONFIG_DIR} + [ ! -d ${SALT_CACHE_DIR} ] && mkdir -p ${SALT_CACHE_DIR} + + echo "base:" > ${SALT_FILE_DIR}/top.sls + for pillar in ${PILLARDIR}/*.sls; do + state_name=$(basename ${pillar%.sls}) + echo -e " ${state_name}:\n - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls + done + + cat << EOF > ${SALT_CONFIG_DIR}/minion +file_client: local +cachedir: ${SALT_CACHE_DIR} +verify_env: False + +file_roots: + base: + - ${SALT_FILE_DIR} + - ${CURDIR}/.. + +pillar_roots: + base: + - ${SALT_PILLAR_DIR} + - ${PILLARDIR} +EOF +} + +fetch_dependency() { + dep_root="${DEPSDIR}/$(basename $1 .git)" + dep_metadata="${dep_root}/metadata.yml" + + [ -d $dep_root ] && log_info "Dependency $1 already fetched" && return 0 + + log_info "Fetching dependency $1" + [ ! -d ${DEPSDIR} ] && mkdir -p ${DEPSDIR} + git clone $1 ${DEPSDIR}/$(basename $1 .git) + + dep_name=$(cat $dep_metadata | python -c "import sys,yaml; print yaml.load(sys.stdin)['name']") + ln -s ${dep_root}/${dep_name} ${SALT_FILE_DIR}/${dep_name} + + METADATA="${dep_metadata}" install_dependencies +} + +install_dependencies() { + grep -E "^dependencies:" ${METADATA} >/dev/null || return 0 + (python - | while read dep; do fetch_dependency "$dep"; done) << EOF +import sys,yaml +for dep in yaml.load(open('${METADATA}', 'ro'))['dependencies']: + print dep["source"] +EOF +} + +clean() { + log_info "Cleaning up ${BUILDDIR}" + [ -d ${BUILDDIR} ] && rm -rf ${BUILDDIR} || exit 0 +} + +salt_run() { + source ${VENV_DIR}/bin/activate + salt-call ${SALT_OPTS} $* +} + +prepare() { + [ -d ${BUILDDIR} ] && mkdir -p ${BUILDDIR} + + setup_virtualenv + setup_pillar + setup_salt + install_dependencies +} + +run() { + for pillar in ${PILLARDIR}/*.sls; do + state_name=$(basename ${pillar%.sls}) + salt_run --id=${state_name} state.show_sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1) + done +} + +_atexit() { + RETVAL=$? + trap true INT TERM EXIT + + if [ $RETVAL -ne 0 ]; then + log_err "Execution failed" + else + log_info "Execution successful" + fi + return $RETVAL +} + +## Main +trap _atexit INT TERM EXIT + +case $1 in + clean) + clean + ;; + prepare) + prepare + ;; + run) + run + ;; + *) + prepare + run + ;; +esac