Initial add of messaging wrapper cookbook

We are getting to a point where we will need this.  This is a first
round attempt at putting together a rabbit cookbook.  It enables
clustering rabbit by default, since this is our (AT&T) current reference
arch.  It uses data bags for encrypted secrets much like the existing
stackforge cookbooks do.  Also, upgraded to new Berkshelf which handles
locking to cookbooks properly.

Change-Id: I0edfc996df413c6e41a31d53f964d7f5fc9e0aeb
This commit is contained in:
John Dewey 2013-06-22 02:12:04 -07:00
parent 902add1ca8
commit 4b9493680f
16 changed files with 648 additions and 10 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
.bundle/

4
.gitreview Normal file
View File

@ -0,0 +1,4 @@
[gerrit]
host=review.openstack.org
port=29418
project=stackforge/cookbook-openstack-ops-messaging.git

25
.tailor Normal file
View File

@ -0,0 +1,25 @@
Tailor.config do |config|
config.formatters "text"
config.file_set '**/*.rb' do |style|
style.max_line_length 80, level: :off
style.allow_camel_case_methods false, level: :error
style.allow_hard_tabs false, level: :error
style.allow_screaming_snake_case_classes false, level: :error
style.allow_trailing_line_spaces false, level: :error
style.allow_invalid_ruby false, level: :warn
style.indentation_spaces 2, level: :error
style.max_code_lines_in_class 300, level: :error
style.max_code_lines_in_method 30, level: :error
style.spaces_after_comma 1, level: :error
style.spaces_after_lbrace 1, level: :error
style.spaces_after_lbracket 0, level: :error
style.spaces_after_lparen 0, level: :error
style.spaces_before_comma 0, level: :error
style.spaces_before_lbrace 1, level: :error
style.spaces_before_rbrace 1, level: :error
style.spaces_before_rbracket 0, level: :error
style.spaces_before_rparen 0, level: :error
style.spaces_in_empty_braces 0, level: :error
style.trailing_newlines 1, level: :error
end
end

4
Berksfile Normal file
View File

@ -0,0 +1,4 @@
metadata
cookbook "openstack-common",
git: "git://github.com/stackforge/cookbook-openstack-common.git"

46
Berksfile.lock Normal file
View File

@ -0,0 +1,46 @@
{
"sha": "db43c403a1eba40c4c3e7833bc8586191ed34ac7",
"sources": {
"openstack-ops-messaging": {
"path": "."
},
"openstack-common": {
"locked_version": "0.2.2",
"git": "git://github.com/stackforge/cookbook-openstack-common.git",
"ref": "0c771cfab3147489e7dffcf9b9b067f2aa9d0f8b"
},
"rabbitmq": {
"locked_version": "2.1.0"
},
"erlang": {
"locked_version": "1.3.0"
},
"apt": {
"locked_version": "1.10.0"
},
"yum": {
"locked_version": "2.2.2"
},
"build-essential": {
"locked_version": "1.4.0"
},
"database": {
"locked_version": "1.4.0"
},
"mysql": {
"locked_version": "3.0.0"
},
"openssl": {
"locked_version": "1.0.2"
},
"postgresql": {
"locked_version": "3.0.0"
},
"aws": {
"locked_version": "0.101.0"
},
"xfs": {
"locked_version": "1.1.0"
}
}
}

9
Gemfile Normal file
View File

@ -0,0 +1,9 @@
source "https://rubygems.org"
gem "chef", "~> 11.4.4"
gem "json", "<= 1.7.7" # chef 11 dependency
gem "berkshelf", "~> 2.0.3"
gem "chefspec", "~> 1.3.0"
gem "foodcritic"
gem "strainer"
gem "tailor"

223
Gemfile.lock Normal file
View File

@ -0,0 +1,223 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (4.0.0)
i18n (~> 0.6, >= 0.6.4)
minitest (~> 4.2)
multi_json (~> 1.3)
thread_safe (~> 0.1)
tzinfo (~> 0.3.37)
addressable (2.3.4)
akami (1.2.0)
gyoku (>= 0.4.0)
nokogiri (>= 1.4.0)
atomic (1.1.9)
berkshelf (2.0.5)
activesupport (>= 3.2.0)
addressable (~> 2.3.4)
buff-shell_out (~> 0.1)
celluloid (>= 0.14.0)
chozo (>= 0.6.1)
faraday (>= 0.8.5)
hashie (>= 2.0.2)
minitar (~> 0.5.4)
rbzip2 (~> 0.2.0)
retryable (~> 1.3.3)
ridley (~> 1.2.1)
solve (>= 0.5.0)
thor (~> 0.18.0)
buff-extensions (0.5.0)
buff-ruby_engine (0.1.0)
buff-shell_out (0.1.0)
buff-ruby_engine (~> 0.1.0)
builder (3.2.2)
celluloid (0.14.1)
timers (>= 1.0.0)
celluloid-io (0.14.1)
celluloid (>= 0.14.1)
nio4r (>= 0.4.5)
chef (11.4.4)
erubis
highline (>= 1.6.9)
json (>= 1.4.4, <= 1.7.7)
mixlib-authentication (>= 1.3.0)
mixlib-cli (~> 1.3.0)
mixlib-config (>= 1.1.2)
mixlib-log (>= 1.3.0)
mixlib-shellout
net-ssh (~> 2.6)
net-ssh-multi (~> 1.1.0)
ohai (>= 0.6.0)
rest-client (>= 1.0.4, < 1.7.0)
yajl-ruby (~> 1.1)
chefspec (1.3.1)
chef (>= 10.0)
erubis
fauxhai (>= 0.1.1, < 2.0)
minitest-chef-handler (>= 0.6.0)
rspec (~> 2.0)
chozo (0.6.1)
activesupport (>= 3.2.0)
hashie (>= 2.0.2)
multi_json (>= 1.3.0)
ci_reporter (1.8.4)
builder (>= 2.1.2)
diff-lcs (1.2.4)
erubis (2.7.0)
faraday (0.8.7)
multipart-post (~> 1.1)
fauxhai (1.1.1)
httparty
net-ssh
ohai
ffi (1.9.0)
foodcritic (2.1.0)
erubis
gherkin (~> 2.11.7)
nokogiri (~> 1.5.4)
rak (~> 1.4)
treetop (~> 1.4.10)
yajl-ruby (~> 1.1.0)
gherkin (2.11.8)
multi_json (~> 1.3)
gssapi (1.0.3)
ffi (>= 1.0.1)
gyoku (1.0.0)
builder (>= 2.1.2)
hashie (2.0.5)
highline (1.6.19)
httparty (0.11.0)
multi_json (~> 1.0)
multi_xml (>= 0.5.2)
httpclient (2.2.0.2)
httpi (0.9.7)
rack
i18n (0.6.4)
ipaddress (0.8.0)
json (1.7.7)
little-plugger (1.1.3)
log_switch (0.4.0)
logging (1.6.2)
little-plugger (>= 1.1.3)
mime-types (1.23)
minitar (0.5.4)
minitest (4.7.5)
minitest-chef-handler (1.0.1)
chef
ci_reporter
minitest (~> 4.7.3)
mixlib-authentication (1.3.0)
mixlib-log
mixlib-cli (1.3.0)
mixlib-config (1.1.2)
mixlib-log (1.6.0)
mixlib-shellout (1.1.0)
multi_json (1.7.7)
multi_xml (0.5.4)
multipart-post (1.2.0)
net-http-persistent (2.8)
net-ssh (2.6.7)
net-ssh-gateway (1.2.0)
net-ssh (>= 2.6.5)
net-ssh-multi (1.1)
net-ssh (>= 2.1.4)
net-ssh-gateway (>= 0.99.0)
nio4r (0.4.6)
nokogiri (1.5.10)
nori (1.1.5)
ohai (6.16.0)
ipaddress
mixlib-cli
mixlib-config
mixlib-log
mixlib-shellout
systemu
yajl-ruby
polyglot (0.3.3)
rack (1.5.2)
rak (1.4)
rbzip2 (0.2.0)
rest-client (1.6.7)
mime-types (>= 1.16)
retryable (1.3.3)
ridley (1.2.3)
addressable
buff-extensions (~> 0.3)
buff-shell_out (~> 0.1)
celluloid (~> 0.14.0)
celluloid-io (~> 0.14.0)
erubis
faraday (>= 0.8.4)
hashie (>= 2.0.2)
json (>= 1.7.7)
mixlib-authentication (>= 1.3.0)
net-http-persistent (>= 2.8)
net-ssh
retryable
solve (>= 0.4.4)
varia_model (~> 0.1)
winrm (~> 1.1.0)
rspec (2.13.0)
rspec-core (~> 2.13.0)
rspec-expectations (~> 2.13.0)
rspec-mocks (~> 2.13.0)
rspec-core (2.13.1)
rspec-expectations (2.13.0)
diff-lcs (>= 1.1.3, < 2.0)
rspec-mocks (2.13.1)
rubyntlm (0.1.1)
savon (0.9.5)
akami (~> 1.0)
builder (>= 2.1.2)
gyoku (>= 0.4.0)
httpi (~> 0.9)
nokogiri (>= 1.4.0)
nori (~> 1.0)
wasabi (~> 1.0)
solve (0.5.0)
strainer (3.0.3)
berkshelf (~> 2.0)
systemu (2.5.2)
tailor (1.2.1)
log_switch (>= 0.3.0)
term-ansicolor (>= 1.0.5)
text-table (>= 1.2.2)
term-ansicolor (1.2.2)
tins (~> 0.8)
text-table (1.2.3)
thor (0.18.1)
thread_safe (0.1.0)
atomic
timers (1.1.0)
tins (0.8.2)
treetop (1.4.14)
polyglot
polyglot (>= 0.3.1)
tzinfo (0.3.37)
uuidtools (2.1.4)
varia_model (0.1.0)
buff-extensions (~> 0.1)
hashie (>= 2.0.2)
wasabi (1.0.0)
nokogiri (>= 1.4.0)
winrm (1.1.2)
gssapi (~> 1.0.0)
httpclient (~> 2.2.0.2)
logging (~> 1.6.1)
nokogiri (~> 1.5.0)
rubyntlm (~> 0.1.1)
savon (= 0.9.5)
uuidtools (~> 2.1.2)
yajl-ruby (1.1.0)
PLATFORMS
ruby
DEPENDENCIES
berkshelf (~> 2.0.3)
chef (~> 11.4.4)
chefspec (~> 1.3.0)
foodcritic
json (<= 1.7.7)
strainer
tailor

View File

@ -14,12 +14,21 @@ Chef 11 with Ruby 1.9.x required.
The following cookbooks are dependencies:
* openstack-common
* rabbitmq
# Usage #
The usage of this cookbook is optional, you may choose to set up your own messaging service without using this cookbook. If you choose to do so, you will need to provide all of the attributes listed under the [Attributes](#attributes).
# Resources/Providers #
None
# Templates #
None
# Recipes #
## default ##
@ -28,26 +37,35 @@ None
# Attributes #
# Templates #
# Attributes #
None
* `openstack["mq"]["server_role"]` - which role should other nodes search on to find the messaging service, defaults to 'os-ops-messaging'
* `openstack["mq"]["service_type"]` - which service to use, defaults to 'rabbit'
* `openstack["mq"]["bind_interface"]` - bind to interfaces IPv4 address
Testing
=====
This cookbook uses [bundler](http://gembundler.com/), [berkshelf](http://berkshelf.com/), and [strainer](https://github.com/customink/strainer) to isolate dependencies and run tests.
Tests are defined in Strainerfile.
To run tests:
$ bundle install # install gem dependencies
$ bundle exec berks install # install cookbook dependencies
$ bundle exec strainer test # run tests
License and Author
==================
| | |
|:---------------------|:---------------------------------------------------|
| **Author** | Justin Shepherd (<justin.shepherd@rackspace.com>) |
| **Author** | Jason Cannavale (<jason.cannavale@rackspace.com>) |
| **Author** | Ron Pedde (<ron.pedde@rackspace.com>) |
| **Author** | Joseph Breu (<joseph.breu@rackspace.com>) |
| **Author** | William Kelly (<william.kelly@rackspace.com>) |
| **Author** | Darren Birkett (<darren.birkett@rackspace.co.uk>) |
| **Author** | Evan Callicoat (<evan.callicoat@rackspace.com>) |
| **Author** | Matt Ray (<matt@opscode.com>) |
| **Author** | John Dewey (<jdewey@att.com>) |
| | |
| **Copyright** | Copyright (c) 2012-2013, Rackspace US, Inc. |
| **Copyright** | Copyright (c) 2012-2013, Opscode, Inc. |
| **Copyright** | Copyright (c) 2013, AT&T Services, Inc. |
Licensed under the Apache License, Version 2.0 (the "License");

5
Strainerfile Normal file
View File

@ -0,0 +1,5 @@
# Strainerfile
tailor: bundle exec tailor
knife test: bundle exec knife cookbook test $COOKBOOK
foodcritic: bundle exec foodcritic -f any -t ~FC003 -t ~FC023 $SANDBOX/$COOKBOOK
chefspec: bundle exec rspec $SANDBOX/$COOKBOOK/spec

22
attributes/default.rb Normal file
View File

@ -0,0 +1,22 @@
#
# Cookbook Name:: openstack-ops-messaging
# Recipe:: default
#
# Copyright 2013, AT&T Services, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
default["openstack"]["mq"]["server_role"] = "os-ops-messaging"
default["openstack"]["mq"]["service_type"] = "rabbit"
default["openstack"]["mq"]["bind_interface"] = "lo"

16
metadata.rb Normal file
View File

@ -0,0 +1,16 @@
name "openstack-ops-messaging"
maintainer "AT&T Services, Inc."
maintainer_email "cookbooks@lists.tfoundry.com"
license "Apache 2.0"
description "Provides the shared messaging configuration for Chef for OpenStack."
version "7.0.0"
recipe "server", "Installs and configures server packages for the messaging used by the deployment."
recipe "rabbit-server", "Installs and configured rabbit server packages."
%w{ fedora ubuntu redhat centos }.each do |os|
supports os
end
depends "openstack-common", "~> 0.3.0"
depends "rabbitmq", ">= 2.1.0"

81
recipes/rabbit-server.rb Normal file
View File

@ -0,0 +1,81 @@
#
# Cookbook Name:: openstack-ops-messaging
# Recipe:: mysql-server
#
# Copyright 2013, AT&T Services, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class ::Chef::Recipe
include ::Openstack
end
rabbit_server_role = node["openstack"]["mq"]["server_role"]
user = node["openstack"]["mq"]["user"]
pass = user_password user
cookie = service_password "rabbit_cookie"
vhost = node["openstack"]["mq"]["vhost"]
bind_interface = node["openstack"]["mq"]["bind_interface"]
listen_address = address_for node["openstack"]["mq"]["bind_interface"]
# Used by OpenStack#rabbit_servers/#rabbit_server
node.set["openstack"]["mq"]["listen"] = listen_address
node.override["rabbitmq"]["address"] = listen_address
node.override["rabbitmq"]["default_user"] = user
node.override["rabbitmq"]["default_pass"] = pass
node.override["rabbitmq"]["erlang_cookie"] = cookie
node.override["rabbitmq"]["cluster"] = true
qs = "roles:#{rabbit_server_role} AND environment:#{node.chef_environment}"
node.override["rabbitmq"]["cluster_disk_nodes"] = search(:node, qs).map do |n|
"#{user}@#{n['hostname']}"
end
include_recipe "rabbitmq"
include_recipe "rabbitmq::mgmt_console"
rabbitmq_user "guest" do
action :delete
end
rabbitmq_vhost vhost do
action :add
end
rabbitmq_user user do
password pass
action :add
end
rabbitmq_user user do
vhost vhost
permissions '.* .* .*'
action :set_permissions
end
# Remove the mnesia database. This is necessary so the nodes
# in the cluster will be able to recognize one another.
# TODO(retr0h): This should be handled upstream.
execute "Reset mnesia" do
cwd "/var/lib/rabbitmq"
command <<-EOH.gsub(/^\s+/, "")
service rabbitmq-server stop;
rm -rf mnesia/;
touch .reset_mnesia_database;
service rabbitmq-server start
EOH
not_if { ::File.exists? "/var/lib/rabbitmq/.reset_mnesia_database" }
end

20
recipes/server.rb Normal file
View File

@ -0,0 +1,20 @@
#
# Cookbook Name:: openstack-ops-messaging
# Recipe:: server
#
# Copyright 2013, AT&T Services, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
include_recipe "openstack-ops-messaging::#{node["openstack"]["mq"]["service_type"]}-server"

119
spec/rabbit-server_spec.rb Normal file
View File

@ -0,0 +1,119 @@
require_relative "spec_helper"
describe "openstack-ops-messaging::rabbit-server" do
before { ops_messaging_stubs }
describe "ubuntu" do
before do
@chef_run = ::ChefSpec::ChefRunner.new(::UBUNTU_OPTS) do |n|
n.set["openstack"]["mq"] = {
"user" => "rabbit-user",
"vhost" => "/test-vhost"
}
end
@chef_run.converge "openstack-ops-messaging::rabbit-server"
end
it "overrides default rabbit attributes" do
expect(@chef_run.node["openstack"]["mq"]["listen"]).to eql "127.0.0.1"
expect(@chef_run.node["rabbitmq"]["address"]).to eql "127.0.0.1"
expect(@chef_run.node["rabbitmq"]["default_user"]).to eql "rabbit-user"
expect(@chef_run.node['rabbitmq']['default_pass']).to eql "rabbit-pass"
expect(@chef_run.node['rabbitmq']['erlang_cookie']).to eql(
"erlang-cookie"
)
expect(@chef_run.node['rabbitmq']['cluster']).to be_true
expect(@chef_run.node['rabbitmq']['cluster_disk_nodes']).to eql(
["rabbit-user@host1", "rabbit-user@host2"]
)
end
it "includes rabbit recipes" do
expect(@chef_run).to include_recipe "rabbitmq"
expect(@chef_run).to include_recipe "rabbitmq::mgmt_console"
end
describe "lwrps" do
it "deletes guest user" do
resource = @chef_run.find_resource(
"rabbitmq_user",
"guest"
).to_hash
expect(resource).to include(
:action => [:delete]
)
end
it "adds vhost" do
resource = @chef_run.find_resource(
"rabbitmq_vhost",
"/test-vhost"
).to_hash
expect(resource).to include(
:action => [:add]
)
end
it "adds user" do
resource = @chef_run.find_resource(
"rabbitmq_user",
"rabbit-user"
).to_hash
expect(resource).to include(
:password => "rabbit-pass",
:action => [:add]
)
end
it "sets user permissions" do
resource = @chef_run.find_resource(
"rabbitmq_user",
"rabbit-user"
).to_hash
#expect(resource).to include(
#:vhost => "/test-vhost",
#:permissions => '.* .* .*',
#:action => [:set_permissions]
#)
end
end
describe "mnesia" do
before do
::File.stub(:exists?).and_call_original
opts = ::UBUNTU_OPTS.merge(:evaluate_guards => true)
@chef_run = ::ChefSpec::ChefRunner.new opts
@cmd = <<-EOH.gsub(/^\s+/, "")
service rabbitmq-server stop;
rm -rf mnesia/;
touch .reset_mnesia_database;
service rabbitmq-server start
EOH
@file = "/var/lib/rabbitmq/.reset_mnesia_database"
end
it "resets database" do
::File.should_receive(:exists?).
with(@file).
and_return(false)
@chef_run.converge "openstack-ops-messaging::rabbit-server"
expect(@chef_run).to execute_command(@cmd).with(
:cwd => "/var/lib/rabbitmq"
)
end
it "doesn't reset database when already did" do
::File.should_receive(:exists?).
with(@file).
and_return(true)
@chef_run.converge "openstack-ops-messaging::rabbit-server"
expect(@chef_run).not_to execute_command(@cmd)
end
end
end
end

14
spec/server_spec.rb Normal file
View File

@ -0,0 +1,14 @@
require_relative "spec_helper"
describe "openstack-ops-messaging::server" do
before { ops_messaging_stubs }
describe "ubuntu" do
it "uses proper messaging server recipe" do
chef_run = ::ChefSpec::ChefRunner.new ::UBUNTU_OPTS
chef_run.converge "openstack-ops-messaging::server"
expect(chef_run).to include_recipe "openstack-ops-messaging::rabbit-server"
end
end
end

31
spec/spec_helper.rb Normal file
View File

@ -0,0 +1,31 @@
require "chefspec"
::LOG_LEVEL = :fatal
::REDHAT_OPTS = {
:platform => "redhat",
:version => "6.3",
:log_level => ::LOG_LEVEL
}
::UBUNTU_OPTS = {
:platform => "ubuntu",
:version => "12.04",
:log_level => ::LOG_LEVEL
}
def ops_messaging_stubs
::Chef::Recipe.any_instance.stub(:address_for).
with("lo").
and_return "127.0.0.1"
::Chef::Recipe.any_instance.stub(:search).
with(:node, "roles:os-ops-messaging AND environment:_default").
and_return [
{ 'hostname' => 'host1' },
{ 'hostname' => 'host2' }
]
::Chef::Recipe.any_instance.stub(:user_password).
#with("rabbit-user"). #TODO(retr0h): uncomment once attributes are in openstack-common.
and_return "rabbit-pass"
::Chef::Recipe.any_instance.stub(:service_password).
with("rabbit_cookie").
and_return "erlang-cookie"
end