Initial add of messaging wrapper cookbook

We are getting to a point where we will need this.  This is a first
round attempt at putting together a rabbit cookbook.  It enables
clustering rabbit by default, since this is our (AT&T) current reference
arch.  It uses data bags for encrypted secrets much like the existing
stackforge cookbooks do.

Change-Id: I0edfc996df413c6e41a31d53f964d7f5fc9e0aeb
This commit is contained in:
John Dewey 2013-06-22 02:12:04 -07:00
parent 902add1ca8
commit 0cc211c1cd
16 changed files with 672 additions and 11 deletions

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
.bundle/
Berksfile.lock

4
.gitreview Normal file
View File

@ -0,0 +1,4 @@
[gerrit]
host=review.openstack.org
port=29418
project=stackforge/cookbook-openstack-ops-messaging.git

25
.tailor Normal file
View File

@ -0,0 +1,25 @@
Tailor.config do |config|
config.formatters "text"
config.file_set '**/*.rb' do |style|
style.max_line_length 80, level: :off
style.allow_camel_case_methods false, level: :error
style.allow_hard_tabs false, level: :error
style.allow_screaming_snake_case_classes false, level: :error
style.allow_trailing_line_spaces false, level: :error
style.allow_invalid_ruby false, level: :warn
style.indentation_spaces 2, level: :error
style.max_code_lines_in_class 300, level: :error
style.max_code_lines_in_method 30, level: :error
style.spaces_after_comma 1, level: :error
style.spaces_after_lbrace 1, level: :error
style.spaces_after_lbracket 0, level: :error
style.spaces_after_lparen 0, level: :error
style.spaces_before_comma 0, level: :error
style.spaces_before_lbrace 1, level: :error
style.spaces_before_rbrace 1, level: :error
style.spaces_before_rbracket 0, level: :error
style.spaces_before_rparen 0, level: :error
style.spaces_in_empty_braces 0, level: :error
style.trailing_newlines 1, level: :error
end
end

4
Berksfile Normal file
View File

@ -0,0 +1,4 @@
metadata
cookbook "openstack-common",
git: "git://github.com/stackforge/cookbook-openstack-common.git"

9
Gemfile Normal file
View File

@ -0,0 +1,9 @@
source "https://rubygems.org"
gem "chef", "~> 11.4.4"
gem "json", "<= 1.7.7" # chef 11 dependency
gem "berkshelf", "~> 1.4.5"
gem "chefspec", "~> 1.3.0"
gem "foodcritic"
gem "strainer"
gem "tailor"

206
Gemfile.lock Normal file
View File

@ -0,0 +1,206 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (3.2.13)
i18n (= 0.6.1)
multi_json (~> 1.0)
addressable (2.3.4)
akami (1.2.0)
gyoku (>= 0.4.0)
nokogiri (>= 1.4.0)
berkshelf (1.4.6)
activesupport (>= 3.2.0)
addressable
celluloid (>= 0.14.0)
chozo (>= 0.6.1)
faraday (>= 0.8.5)
hashie (>= 2.0.2)
json (>= 1.5.0)
minitar
mixlib-config (~> 1.1)
mixlib-shellout (~> 1.1)
multi_json (~> 1.5)
retryable
ridley (~> 0.12.4)
solve (>= 0.4.2)
thor (~> 0.18.0)
yajl-ruby
builder (3.2.2)
celluloid (0.14.1)
timers (>= 1.0.0)
chef (11.4.4)
erubis
highline (>= 1.6.9)
json (>= 1.4.4, <= 1.7.7)
mixlib-authentication (>= 1.3.0)
mixlib-cli (~> 1.3.0)
mixlib-config (>= 1.1.2)
mixlib-log (>= 1.3.0)
mixlib-shellout
net-ssh (~> 2.6)
net-ssh-multi (~> 1.1.0)
ohai (>= 0.6.0)
rest-client (>= 1.0.4, < 1.7.0)
yajl-ruby (~> 1.1)
chefspec (1.3.1)
chef (>= 10.0)
erubis
fauxhai (>= 0.1.1, < 2.0)
minitest-chef-handler (>= 0.6.0)
rspec (~> 2.0)
chozo (0.6.1)
activesupport (>= 3.2.0)
hashie (>= 2.0.2)
multi_json (>= 1.3.0)
ci_reporter (1.8.4)
builder (>= 2.1.2)
diff-lcs (1.2.4)
erubis (2.7.0)
faraday (0.8.7)
multipart-post (~> 1.1)
fauxhai (1.1.1)
httparty
net-ssh
ohai
ffi (1.9.0)
foodcritic (2.1.0)
erubis
gherkin (~> 2.11.7)
nokogiri (~> 1.5.4)
rak (~> 1.4)
treetop (~> 1.4.10)
yajl-ruby (~> 1.1.0)
gherkin (2.11.8)
multi_json (~> 1.3)
gssapi (1.0.3)
ffi (>= 1.0.1)
gyoku (1.0.0)
builder (>= 2.1.2)
hashie (2.0.5)
highline (1.6.19)
httparty (0.11.0)
multi_json (~> 1.0)
multi_xml (>= 0.5.2)
httpclient (2.2.0.2)
httpi (0.9.7)
rack
i18n (0.6.1)
ipaddress (0.8.0)
json (1.7.7)
little-plugger (1.1.3)
log_switch (0.4.0)
logging (1.6.2)
little-plugger (>= 1.1.3)
mime-types (1.23)
minitar (0.5.4)
minitest (4.7.5)
minitest-chef-handler (1.0.1)
chef
ci_reporter
minitest (~> 4.7.3)
mixlib-authentication (1.3.0)
mixlib-log
mixlib-cli (1.3.0)
mixlib-config (1.1.2)
mixlib-log (1.6.0)
mixlib-shellout (1.1.0)
multi_json (1.7.7)
multi_xml (0.5.4)
multipart-post (1.2.0)
net-http-persistent (2.8)
net-ssh (2.6.7)
net-ssh-gateway (1.2.0)
net-ssh (>= 2.6.5)
net-ssh-multi (1.1)
net-ssh (>= 2.1.4)
net-ssh-gateway (>= 0.99.0)
nokogiri (1.5.10)
nori (1.1.5)
ohai (6.16.0)
ipaddress
mixlib-cli
mixlib-config
mixlib-log
mixlib-shellout
systemu
yajl-ruby
polyglot (0.3.3)
rack (1.5.2)
rak (1.4)
rest-client (1.6.7)
mime-types (>= 1.16)
retryable (1.3.3)
ridley (0.12.4)
addressable
celluloid (~> 0.14.0)
chozo (>= 0.6.0)
erubis
faraday (>= 0.8.4)
hashie (>= 2.0.2)
mixlib-authentication (>= 1.3.0)
mixlib-config (>= 1.1.0)
mixlib-log (>= 1.3.0)
mixlib-shellout (>= 1.1.0)
net-http-persistent (>= 2.8)
net-ssh
retryable
solve (>= 0.4.4)
winrm (~> 1.1.0)
rspec (2.13.0)
rspec-core (~> 2.13.0)
rspec-expectations (~> 2.13.0)
rspec-mocks (~> 2.13.0)
rspec-core (2.13.1)
rspec-expectations (2.13.0)
diff-lcs (>= 1.1.3, < 2.0)
rspec-mocks (2.13.1)
rubyntlm (0.1.1)
savon (0.9.5)
akami (~> 1.0)
builder (>= 2.1.2)
gyoku (>= 0.4.0)
httpi (~> 0.9)
nokogiri (>= 1.4.0)
nori (~> 1.0)
wasabi (~> 1.0)
solve (0.5.0)
strainer (2.1.0)
berkshelf (~> 1.3)
systemu (2.5.2)
tailor (1.2.1)
log_switch (>= 0.3.0)
term-ansicolor (>= 1.0.5)
text-table (>= 1.2.2)
term-ansicolor (1.2.2)
tins (~> 0.8)
text-table (1.2.3)
thor (0.18.1)
timers (1.1.0)
tins (0.8.2)
treetop (1.4.14)
polyglot
polyglot (>= 0.3.1)
uuidtools (2.1.4)
wasabi (1.0.0)
nokogiri (>= 1.4.0)
winrm (1.1.2)
gssapi (~> 1.0.0)
httpclient (~> 2.2.0.2)
logging (~> 1.6.1)
nokogiri (~> 1.5.0)
rubyntlm (~> 0.1.1)
savon (= 0.9.5)
uuidtools (~> 2.1.2)
yajl-ruby (1.1.0)
PLATFORMS
ruby
DEPENDENCIES
berkshelf (~> 1.4.5)
chef (~> 11.4.4)
chefspec (~> 1.3.0)
foodcritic
json (<= 1.7.7)
strainer
tailor

View File

@ -4,7 +4,8 @@ This cookbook provides shared messaging configuration for the OpenStack **Grizzl
# Requirements #
Chef 11 with Ruby 1.9.x required.
* Chef 11 with Ruby 1.9.x required.
* [Network Addr](https://gist.github.com/jtimberman/1040543) Ohai plugin.
# Platforms #
@ -14,12 +15,21 @@ Chef 11 with Ruby 1.9.x required.
The following cookbooks are dependencies:
* openstack-common
* rabbitmq
# Usage #
The usage of this cookbook is optional, you may choose to set up your own messaging service without using this cookbook. If you choose to do so, you will need to provide all of the attributes listed under the [Attributes](#attributes).
# Resources/Providers #
None
# Templates #
None
# Recipes #
## default ##
@ -28,26 +38,35 @@ None
# Attributes #
# Templates #
# Attributes #
None
* `openstack["mq"]["server_role"]` - which role should other nodes search on to find the messaging service, defaults to 'os-ops-messaging'
* `openstack["mq"]["service_type"]` - which service to use, defaults to 'rabbit'
* `openstack["mq"]["bind_interface"]` - bind to interfaces IPv4 address
Testing
=====
This cookbook uses [bundler](http://gembundler.com/), [berkshelf](http://berkshelf.com/), and [strainer](https://github.com/customink/strainer) to isolate dependencies and run tests.
Tests are defined in Strainerfile.
To run tests:
$ bundle install # install gem dependencies
$ bundle exec berks install # install cookbook dependencies
$ bundle exec strainer test # run tests
License and Author
==================
| | |
|:---------------------|:---------------------------------------------------|
| **Author** | Justin Shepherd (<justin.shepherd@rackspace.com>) |
| **Author** | Jason Cannavale (<jason.cannavale@rackspace.com>) |
| **Author** | Ron Pedde (<ron.pedde@rackspace.com>) |
| **Author** | Joseph Breu (<joseph.breu@rackspace.com>) |
| **Author** | William Kelly (<william.kelly@rackspace.com>) |
| **Author** | Darren Birkett (<darren.birkett@rackspace.co.uk>) |
| **Author** | Evan Callicoat (<evan.callicoat@rackspace.com>) |
| **Author** | Matt Ray (<matt@opscode.com>) |
| **Author** | John Dewey (<jdewey@att.com>) |
| | |
| **Copyright** | Copyright (c) 2012-2013, Rackspace US, Inc. |
| **Copyright** | Copyright (c) 2012-2013, Opscode, Inc. |
| **Copyright** | Copyright (c) 2013, AT&T Services, Inc. |
Licensed under the Apache License, Version 2.0 (the "License");

5
Strainerfile Normal file
View File

@ -0,0 +1,5 @@
# Strainerfile
tailor: bundle exec tailor
knife test: bundle exec knife cookbook test $COOKBOOK
foodcritic: bundle exec foodcritic -f any -t ~FC003 -t ~FC023 $SANDBOX/$COOKBOOK
chefspec: bundle exec rspec $SANDBOX/$COOKBOOK/spec

22
attributes/default.rb Normal file
View File

@ -0,0 +1,22 @@
#
# Cookbook Name:: openstack-ops-messaging
# Recipe:: default
#
# Copyright 2013, AT&T Services, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
default["openstack"]["mq"]["server_role"] = "os-ops-messaging"
default["openstack"]["mq"]["service_type"] = "rabbit"
default["openstack"]["mq"]["bind_interface"] = "lo"

16
metadata.rb Normal file
View File

@ -0,0 +1,16 @@
name "openstack-ops-messaging"
maintainer "AT&T Services, Inc."
maintainer_email "cookbooks@lists.tfoundry.com"
license "Apache 2.0"
description "Provides the shared messaging configuration for Chef for OpenStack."
version "7.0.0"
recipe "server", "Installs and configures server packages for the messaging used by the deployment."
recipe "rabbit-server", "Installs and configured rabbit server packages."
%w{ fedora ubuntu redhat centos }.each do |os|
supports os
end
depends "openstack-common", "~> 0.2.2"
depends "rabbitmq", ">= 2.1.0"

84
recipes/default.rb Normal file
View File

@ -0,0 +1,84 @@
#
# Cookbook Name:: infra-messaging
# Recipe:: default
#
# Copyright 2012, John Dewey
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class ::Chef::Recipe
include ::Openstack
end
rabbit_server_role = node["openstack"]["compute"]["rabbit_server_chef_role"]
user = node["openstack"]["compute"]["rabbit"]["username"]
pass = user_password user
cookie = service_password "rabbit_cookie"
vhost = node["openstack"]["compute"]["rabbit"]["vhost"]
node.override["rabbitmq"]["default_user"] = user
node.override["rabbitmq"]["default_pass"] = pass
node.override["rabbitmq"]["erlang_cookie"] = cookie
node.override["rabbitmq"]["cluster"] = true
node.override["rabbitmq"]["cluster_disk_nodes"] = search(:node, "roles:#{rabbit_server_role}").map do |n|
"#{user}@#{n['hostname']}"
end
include_recipe "rabbitmq"
include_recipe "rabbitmq::mgmt_console"
rabbitmq_user "guest" do
action :delete
end
rabbitmq_vhost vhost do
action :add
end
rabbitmq_user user do
password pass
action :add
end
rabbitmq_user user do
vhost vhost
permissions '".*" ".*" ".*"'
action :set_permissions
end
# Necessary for graphing. The guest user
# by default is admin, since we delete
# this user, I don't see too much harm in
# making the rabbit user an administrator.
rabbitmq_user user do
user_tag "administrator"
action :set_user_tags
end
# Remove the mnesia database. This is necessary so the nodes
# in the cluster will be able to recognize one another.
execute "Reset mnesia" do
cwd "/var/lib/rabbitmq"
command <<-EOH
service rabbitmq-server stop;
rm -rf mnesia/;
touch .reset_mnesia_database;
service rabbitmq-server start
EOH
not_if { ::File.exists? "/var/lib/rabbitmq/.reset_mnesia_database" }
end

81
recipes/rabbit-server.rb Normal file
View File

@ -0,0 +1,81 @@
#
# Cookbook Name:: openstack-ops-messaging
# Recipe:: mysql-server
#
# Copyright 2013, AT&T Services, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class ::Chef::Recipe
include ::Openstack
end
rabbit_server_role = node["openstack"]["mq"]["server_role"]
user = node["openstack"]["mq"]["user"]
pass = user_password user
cookie = service_password "rabbit_cookie"
vhost = node["openstack"]["mq"]["vhost"]
bind_interface = node["openstack"]["mq"]["bind_interface"]
listen_address = node["network"]["ipaddress_#{bind_interface}"]
# Used by OpenStack#rabbit_servers/#rabbit_server
node.set["openstack"]["mq"]["listen"] = listen_address
node.override["rabbitmq"]["address"] = listen_address
node.override["rabbitmq"]["default_user"] = user
node.override["rabbitmq"]["default_pass"] = pass
node.override["rabbitmq"]["erlang_cookie"] = cookie
node.override["rabbitmq"]["cluster"] = true
qs = "roles:#{rabbit_server_role} AND environment:#{node.chef_environment}"
node.override["rabbitmq"]["cluster_disk_nodes"] = search(:node, qs).map do |n|
"#{user}@#{n['hostname']}"
end
include_recipe "rabbitmq"
include_recipe "rabbitmq::mgmt_console"
rabbitmq_user "guest" do
action :delete
end
rabbitmq_vhost vhost do
action :add
end
rabbitmq_user user do
password pass
action :add
end
rabbitmq_user user do
vhost vhost
permissions '.* .* .*'
action :set_permissions
end
# Remove the mnesia database. This is necessary so the nodes
# in the cluster will be able to recognize one another.
# TODO(retr0h): This should be handled upstream.
execute "Reset mnesia" do
cwd "/var/lib/rabbitmq"
command <<-EOH.gsub(/^\s+/, "")
service rabbitmq-server stop;
rm -rf mnesia/;
touch .reset_mnesia_database;
service rabbitmq-server start
EOH
not_if { ::File.exists? "/var/lib/rabbitmq/.reset_mnesia_database" }
end

20
recipes/server.rb Normal file
View File

@ -0,0 +1,20 @@
#
# Cookbook Name:: openstack-ops-messaging
# Recipe:: server
#
# Copyright 2013, AT&T Services, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
include_recipe "openstack-ops-messaging::#{node["openstack"]["mq"]["service_type"]}-server"

122
spec/rabbit-server_spec.rb Normal file
View File

@ -0,0 +1,122 @@
require_relative "spec_helper"
describe "openstack-ops-messaging::rabbit-server" do
before { ops_messaging_stubs }
describe "ubuntu" do
before do
@chef_run = ::ChefSpec::ChefRunner.new(::UBUNTU_OPTS) do |n|
n.set["openstack"]["mq"] = {
"user" => "rabbit-user",
"vhost" => "/test-vhost"
}
n.set["network"] = {
"ipaddress_lo" => "127.0.0.1"
}
end
@chef_run.converge "openstack-ops-messaging::rabbit-server"
end
it "overrides default rabbit attributes" do
expect(@chef_run.node["openstack"]["mq"]["listen"]).to eql "127.0.0.1"
expect(@chef_run.node["rabbitmq"]["address"]).to eql "127.0.0.1"
expect(@chef_run.node["rabbitmq"]["default_user"]).to eql "rabbit-user"
expect(@chef_run.node['rabbitmq']['default_pass']).to eql "rabbit-pass"
expect(@chef_run.node['rabbitmq']['erlang_cookie']).to eql(
"erlang-cookie"
)
expect(@chef_run.node['rabbitmq']['cluster']).to be_true
expect(@chef_run.node['rabbitmq']['cluster_disk_nodes']).to eql(
["rabbit-user@host1", "rabbit-user@host2"]
)
end
it "includes rabbit recipes" do
expect(@chef_run).to include_recipe "rabbitmq"
expect(@chef_run).to include_recipe "rabbitmq::mgmt_console"
end
describe "lwrps" do
it "deletes guest user" do
resource = @chef_run.find_resource(
"rabbitmq_user",
"guest"
).to_hash
expect(resource).to include(
:action => [:delete]
)
end
it "adds vhost" do
resource = @chef_run.find_resource(
"rabbitmq_vhost",
"/test-vhost"
).to_hash
expect(resource).to include(
:action => [:add]
)
end
it "adds user" do
resource = @chef_run.find_resource(
"rabbitmq_user",
"rabbit-user"
).to_hash
expect(resource).to include(
:password => "rabbit-pass",
:action => [:add]
)
end
it "sets user permissions" do
resource = @chef_run.find_resource(
"rabbitmq_user",
"rabbit-user"
).to_hash
#expect(resource).to include(
#:vhost => "/test-vhost",
#:permissions => '.* .* .*',
#:action => [:set_permissions]
#)
end
end
describe "mnesia" do
before do
::File.stub(:exists?).and_call_original
opts = ::UBUNTU_OPTS.merge(:evaluate_guards => true)
@chef_run = ::ChefSpec::ChefRunner.new opts
@cmd = <<-EOH.gsub(/^\s+/, "")
service rabbitmq-server stop;
rm -rf mnesia/;
touch .reset_mnesia_database;
service rabbitmq-server start
EOH
@file = "/var/lib/rabbitmq/.reset_mnesia_database"
end
it "resets database" do
::File.should_receive(:exists?).
with(@file).
and_return(false)
@chef_run.converge "openstack-ops-messaging::rabbit-server"
expect(@chef_run).to execute_command(@cmd).with(
:cwd => "/var/lib/rabbitmq"
)
end
it "doesn't reset database when already did" do
::File.should_receive(:exists?).
with(@file).
and_return(true)
@chef_run.converge "openstack-ops-messaging::rabbit-server"
expect(@chef_run).not_to execute_command(@cmd)
end
end
end
end

14
spec/server_spec.rb Normal file
View File

@ -0,0 +1,14 @@
require_relative "spec_helper"
describe "openstack-ops-messaging::server" do
before { ops_messaging_stubs }
describe "ubuntu" do
it "uses proper messaging server recipe" do
chef_run = ::ChefSpec::ChefRunner.new ::UBUNTU_OPTS
chef_run.converge "openstack-ops-messaging::server"
expect(chef_run).to include_recipe "openstack-ops-messaging::rabbit-server"
end
end
end

28
spec/spec_helper.rb Normal file
View File

@ -0,0 +1,28 @@
require "chefspec"
::LOG_LEVEL = :fatal
::REDHAT_OPTS = {
:platform => "redhat",
:version => "6.3",
:log_level => ::LOG_LEVEL
}
::UBUNTU_OPTS = {
:platform => "ubuntu",
:version => "12.04",
:log_level => ::LOG_LEVEL
}
def ops_messaging_stubs
::Chef::Recipe.any_instance.stub(:search).
with(:node, "roles:os-ops-messaging AND environment:_default").
and_return [
{ 'hostname' => 'host1' },
{ 'hostname' => 'host2' }
]
::Chef::Recipe.any_instance.stub(:user_password).
#with("rabbit-user"). #TODO(retr0h): uncomment once attributes are in openstack-common.
and_return "rabbit-pass"
::Chef::Recipe.any_instance.stub(:service_password).
with("rabbit_cookie").
and_return "erlang-cookie"
end