Initial add of messaging wrapper cookbook

We are getting to a point where we will need this.  This is a first
round attempt at putting together a rabbit cookbook.  It enables
clustering rabbit by default, since this is our (AT&T) current reference
arch.  It uses data bags for encrypted secrets much like the existing
stackforge cookbooks do.

Change-Id: I0edfc996df413c6e41a31d53f964d7f5fc9e0aeb
This commit is contained in:
John Dewey 2013-06-22 02:12:04 -07:00
parent 902add1ca8
commit 74d9a90d36
16 changed files with 654 additions and 10 deletions

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
.bundle/
Berksfile.lock

4
.gitreview Normal file
View File

@ -0,0 +1,4 @@
[gerrit]
host=review.openstack.org
port=29418
project=stackforge/cookbook-openstack-ops-messaging.git

25
.tailor Normal file
View File

@ -0,0 +1,25 @@
Tailor.config do |config|
config.formatters "text"
config.file_set '**/*.rb' do |style|
style.max_line_length 80, level: :off
style.allow_camel_case_methods false, level: :error
style.allow_hard_tabs false, level: :error
style.allow_screaming_snake_case_classes false, level: :error
style.allow_trailing_line_spaces false, level: :error
style.allow_invalid_ruby false, level: :warn
style.indentation_spaces 2, level: :error
style.max_code_lines_in_class 300, level: :error
style.max_code_lines_in_method 30, level: :error
style.spaces_after_comma 1, level: :error
style.spaces_after_lbrace 1, level: :error
style.spaces_after_lbracket 0, level: :error
style.spaces_after_lparen 0, level: :error
style.spaces_before_comma 0, level: :error
style.spaces_before_lbrace 1, level: :error
style.spaces_before_rbrace 1, level: :error
style.spaces_before_rbracket 0, level: :error
style.spaces_before_rparen 0, level: :error
style.spaces_in_empty_braces 0, level: :error
style.trailing_newlines 1, level: :error
end
end

4
Berksfile Normal file
View File

@ -0,0 +1,4 @@
metadata
cookbook "openstack-common",
git: "git://github.com/stackforge/cookbook-openstack-common.git"

9
Gemfile Normal file
View File

@ -0,0 +1,9 @@
source "https://rubygems.org"
gem "chef", "~> 11.4.4"
gem "json", "<= 1.7.7" # chef 11 dependency
gem "berkshelf", "~> 1.4.5"
gem "chefspec", "~> 1.3.0"
gem "foodcritic"
gem "strainer"
gem "tailor"

206
Gemfile.lock Normal file
View File

@ -0,0 +1,206 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (3.2.13)
i18n (= 0.6.1)
multi_json (~> 1.0)
addressable (2.3.4)
akami (1.2.0)
gyoku (>= 0.4.0)
nokogiri (>= 1.4.0)
berkshelf (1.4.6)
activesupport (>= 3.2.0)
addressable
celluloid (>= 0.14.0)
chozo (>= 0.6.1)
faraday (>= 0.8.5)
hashie (>= 2.0.2)
json (>= 1.5.0)
minitar
mixlib-config (~> 1.1)
mixlib-shellout (~> 1.1)
multi_json (~> 1.5)
retryable
ridley (~> 0.12.4)
solve (>= 0.4.2)
thor (~> 0.18.0)
yajl-ruby
builder (3.2.2)
celluloid (0.14.1)
timers (>= 1.0.0)
chef (11.4.4)
erubis
highline (>= 1.6.9)
json (>= 1.4.4, <= 1.7.7)
mixlib-authentication (>= 1.3.0)
mixlib-cli (~> 1.3.0)
mixlib-config (>= 1.1.2)
mixlib-log (>= 1.3.0)
mixlib-shellout
net-ssh (~> 2.6)
net-ssh-multi (~> 1.1.0)
ohai (>= 0.6.0)
rest-client (>= 1.0.4, < 1.7.0)
yajl-ruby (~> 1.1)
chefspec (1.3.1)
chef (>= 10.0)
erubis
fauxhai (>= 0.1.1, < 2.0)
minitest-chef-handler (>= 0.6.0)
rspec (~> 2.0)
chozo (0.6.1)
activesupport (>= 3.2.0)
hashie (>= 2.0.2)
multi_json (>= 1.3.0)
ci_reporter (1.8.4)
builder (>= 2.1.2)
diff-lcs (1.2.4)
erubis (2.7.0)
faraday (0.8.7)
multipart-post (~> 1.1)
fauxhai (1.1.1)
httparty
net-ssh
ohai
ffi (1.9.0)
foodcritic (2.1.0)
erubis
gherkin (~> 2.11.7)
nokogiri (~> 1.5.4)
rak (~> 1.4)
treetop (~> 1.4.10)
yajl-ruby (~> 1.1.0)
gherkin (2.11.8)
multi_json (~> 1.3)
gssapi (1.0.3)
ffi (>= 1.0.1)
gyoku (1.0.0)
builder (>= 2.1.2)
hashie (2.0.5)
highline (1.6.19)
httparty (0.11.0)
multi_json (~> 1.0)
multi_xml (>= 0.5.2)
httpclient (2.2.0.2)
httpi (0.9.7)
rack
i18n (0.6.1)
ipaddress (0.8.0)
json (1.7.7)
little-plugger (1.1.3)
log_switch (0.4.0)
logging (1.6.2)
little-plugger (>= 1.1.3)
mime-types (1.23)
minitar (0.5.4)
minitest (4.7.5)
minitest-chef-handler (1.0.1)
chef
ci_reporter
minitest (~> 4.7.3)
mixlib-authentication (1.3.0)
mixlib-log
mixlib-cli (1.3.0)
mixlib-config (1.1.2)
mixlib-log (1.6.0)
mixlib-shellout (1.1.0)
multi_json (1.7.7)
multi_xml (0.5.4)
multipart-post (1.2.0)
net-http-persistent (2.8)
net-ssh (2.6.7)
net-ssh-gateway (1.2.0)
net-ssh (>= 2.6.5)
net-ssh-multi (1.1)
net-ssh (>= 2.1.4)
net-ssh-gateway (>= 0.99.0)
nokogiri (1.5.10)
nori (1.1.5)
ohai (6.16.0)
ipaddress
mixlib-cli
mixlib-config
mixlib-log
mixlib-shellout
systemu
yajl-ruby
polyglot (0.3.3)
rack (1.5.2)
rak (1.4)
rest-client (1.6.7)
mime-types (>= 1.16)
retryable (1.3.3)
ridley (0.12.4)
addressable
celluloid (~> 0.14.0)
chozo (>= 0.6.0)
erubis
faraday (>= 0.8.4)
hashie (>= 2.0.2)
mixlib-authentication (>= 1.3.0)
mixlib-config (>= 1.1.0)
mixlib-log (>= 1.3.0)
mixlib-shellout (>= 1.1.0)
net-http-persistent (>= 2.8)
net-ssh
retryable
solve (>= 0.4.4)
winrm (~> 1.1.0)
rspec (2.13.0)
rspec-core (~> 2.13.0)
rspec-expectations (~> 2.13.0)
rspec-mocks (~> 2.13.0)
rspec-core (2.13.1)
rspec-expectations (2.13.0)
diff-lcs (>= 1.1.3, < 2.0)
rspec-mocks (2.13.1)
rubyntlm (0.1.1)
savon (0.9.5)
akami (~> 1.0)
builder (>= 2.1.2)
gyoku (>= 0.4.0)
httpi (~> 0.9)
nokogiri (>= 1.4.0)
nori (~> 1.0)
wasabi (~> 1.0)
solve (0.5.0)
strainer (2.1.0)
berkshelf (~> 1.3)
systemu (2.5.2)
tailor (1.2.1)
log_switch (>= 0.3.0)
term-ansicolor (>= 1.0.5)
text-table (>= 1.2.2)
term-ansicolor (1.2.2)
tins (~> 0.8)
text-table (1.2.3)
thor (0.18.1)
timers (1.1.0)
tins (0.8.2)
treetop (1.4.14)
polyglot
polyglot (>= 0.3.1)
uuidtools (2.1.4)
wasabi (1.0.0)
nokogiri (>= 1.4.0)
winrm (1.1.2)
gssapi (~> 1.0.0)
httpclient (~> 2.2.0.2)
logging (~> 1.6.1)
nokogiri (~> 1.5.0)
rubyntlm (~> 0.1.1)
savon (= 0.9.5)
uuidtools (~> 2.1.2)
yajl-ruby (1.1.0)
PLATFORMS
ruby
DEPENDENCIES
berkshelf (~> 1.4.5)
chef (~> 11.4.4)
chefspec (~> 1.3.0)
foodcritic
json (<= 1.7.7)
strainer
tailor

View File

@ -14,12 +14,21 @@ Chef 11 with Ruby 1.9.x required.
The following cookbooks are dependencies:
* openstack-common
* rabbitmq
# Usage #
The usage of this cookbook is optional, you may choose to set up your own messaging service without using this cookbook. If you choose to do so, you will need to provide all of the attributes listed under the [Attributes](#attributes).
# Resources/Providers #
None
# Templates #
None
# Recipes #
## default ##
@ -28,26 +37,34 @@ None
# Attributes #
# Templates #
# Attributes #
None
* `openstack['mq']['server_role']` - which role should other nodes search on to find the messaging service, defaults to 'os-ops-messaging'
* `openstack['mq']['service_type']` - which service to use, defaults to 'rabbit'
Testing
=====
This cookbook uses [bundler](http://gembundler.com/), [berkshelf](http://berkshelf.com/), and [strainer](https://github.com/customink/strainer) to isolate dependencies and run tests.
Tests are defined in Strainerfile.
To run tests:
$ bundle install # install gem dependencies
$ bundle exec berks install # install cookbook dependencies
$ bundle exec strainer test # run tests
License and Author
==================
| | |
|:---------------------|:---------------------------------------------------|
| **Author** | Justin Shepherd (<justin.shepherd@rackspace.com>) |
| **Author** | Jason Cannavale (<jason.cannavale@rackspace.com>) |
| **Author** | Ron Pedde (<ron.pedde@rackspace.com>) |
| **Author** | Joseph Breu (<joseph.breu@rackspace.com>) |
| **Author** | William Kelly (<william.kelly@rackspace.com>) |
| **Author** | Darren Birkett (<darren.birkett@rackspace.co.uk>) |
| **Author** | Evan Callicoat (<evan.callicoat@rackspace.com>) |
| **Author** | Matt Ray (<matt@opscode.com>) |
| **Author** | John Dewey (<jdewey@att.com>) |
| | |
| **Copyright** | Copyright (c) 2012-2013, Rackspace US, Inc. |
| **Copyright** | Copyright (c) 2012-2013, Opscode, Inc. |
| **Copyright** | Copyright (c) 2013, AT&T Services, Inc. |
Licensed under the Apache License, Version 2.0 (the "License");

5
Strainerfile Normal file
View File

@ -0,0 +1,5 @@
# Strainerfile
tailor: bundle exec tailor
knife test: bundle exec knife cookbook test $COOKBOOK
foodcritic: bundle exec foodcritic -f any -t ~FC003 -t ~FC023 $SANDBOX/$COOKBOOK
chefspec: bundle exec rspec $SANDBOX/$COOKBOOK/spec

21
attributes/default.rb Normal file
View File

@ -0,0 +1,21 @@
#
# Cookbook Name:: openstack-ops-messaging
# Recipe:: default
#
# Copyright 2013, AT&T Services, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
default["openstack"]["mq"]["server_role"] = "os-ops-messaging"
default["openstack"]["mq"]["service_type"] = "rabbit"

16
metadata.rb Normal file
View File

@ -0,0 +1,16 @@
name "openstack-ops-messaging"
maintainer "AT&T Services, Inc."
maintainer_email "cookbooks@lists.tfoundry.com"
license "Apache 2.0"
description "Provides the shared messaging configuration for Chef for OpenStack."
version "7.0.0"
recipe "server", "Installs and configures server packages for the messaging used by the deployment."
recipe "rabbit-server", "Installs and configured rabbit server packages."
%w{ fedora ubuntu redhat centos }.each do |os|
supports os
end
depends "openstack-common", "~> 0.2.2"
depends "rabbitmq", ">= 2.1.0"

84
recipes/default.rb Normal file
View File

@ -0,0 +1,84 @@
#
# Cookbook Name:: infra-messaging
# Recipe:: default
#
# Copyright 2012, John Dewey
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class ::Chef::Recipe
include ::Openstack
end
rabbit_server_role = node["openstack"]["compute"]["rabbit_server_chef_role"]
user = node["openstack"]["compute"]["rabbit"]["username"]
pass = user_password user
cookie = service_password "rabbit_cookie"
vhost = node["openstack"]["compute"]["rabbit"]["vhost"]
node.override["rabbitmq"]["default_user"] = user
node.override["rabbitmq"]["default_pass"] = pass
node.override["rabbitmq"]["erlang_cookie"] = cookie
node.override["rabbitmq"]["cluster"] = true
node.override["rabbitmq"]["cluster_disk_nodes"] = search(:node, "roles:#{rabbit_server_role}").map do |n|
"#{user}@#{n['hostname']}"
end
include_recipe "rabbitmq"
include_recipe "rabbitmq::mgmt_console"
rabbitmq_user "guest" do
action :delete
end
rabbitmq_vhost vhost do
action :add
end
rabbitmq_user user do
password pass
action :add
end
rabbitmq_user user do
vhost vhost
permissions '".*" ".*" ".*"'
action :set_permissions
end
# Necessary for graphing. The guest user
# by default is admin, since we delete
# this user, I don't see too much harm in
# making the rabbit user an administrator.
rabbitmq_user user do
user_tag "administrator"
action :set_user_tags
end
# Remove the mnesia database. This is necessary so the nodes
# in the cluster will be able to recognize one another.
execute "Reset mnesia" do
cwd "/var/lib/rabbitmq"
command <<-EOH
service rabbitmq-server stop;
rm -rf mnesia/;
touch .reset_mnesia_database;
service rabbitmq-server start
EOH
not_if { ::File.exists? "/var/lib/rabbitmq/.reset_mnesia_database" }
end

76
recipes/rabbit-server.rb Normal file
View File

@ -0,0 +1,76 @@
#
# Cookbook Name:: openstack-ops-messaging
# Recipe:: mysql-server
#
# Copyright 2013, AT&T Services, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class ::Chef::Recipe
include ::Openstack
end
rabbit_server_role = node["openstack"]["mq"]["server_role"]
user = node["openstack"]["mq"]["user"]
pass = user_password user
cookie = service_password "rabbit_cookie"
vhost = node["openstack"]["mq"]["vhost"]
#TODO(retr0h): Need to set listen address.
node.override["rabbitmq"]["default_user"] = user
node.override["rabbitmq"]["default_pass"] = pass
node.override["rabbitmq"]["erlang_cookie"] = cookie
node.override["rabbitmq"]["cluster"] = true
query = "roles:#{rabbit_server_role} AND environment:#{node.chef_environment}"
node.override["rabbitmq"]["cluster_disk_nodes"] = search(:node, query).map do |n|
"#{user}@#{n['hostname']}"
end
include_recipe "rabbitmq"
include_recipe "rabbitmq::mgmt_console"
rabbitmq_user "guest" do
action :delete
end
rabbitmq_vhost vhost do
action :add
end
rabbitmq_user user do
password pass
action :add
end
rabbitmq_user user do
vhost vhost
permissions '.* .* .*'
action :set_permissions
end
# Remove the mnesia database. This is necessary so the nodes
# in the cluster will be able to recognize one another.
# TODO(retr0h): This should be handled upstream.
execute "Reset mnesia" do
cwd "/var/lib/rabbitmq"
command <<-EOH.gsub(/^\s+/, "")
service rabbitmq-server stop;
rm -rf mnesia/;
touch .reset_mnesia_database;
service rabbitmq-server start
EOH
not_if { ::File.exists? "/var/lib/rabbitmq/.reset_mnesia_database" }
end

20
recipes/server.rb Normal file
View File

@ -0,0 +1,20 @@
#
# Cookbook Name:: openstack-ops-messaging
# Recipe:: server
#
# Copyright 2013, AT&T Services, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
include_recipe "openstack-ops-messaging::#{node["openstack"]["mq"]["service_type"]}-server"

113
spec/rabbit-server_spec.rb Normal file
View File

@ -0,0 +1,113 @@
require_relative "spec_helper"
describe "openstack-ops-messaging::rabbit-server" do
before { ops_messaging_stubs }
describe "ubuntu" do
before do
@chef_run = ::ChefSpec::ChefRunner.new(::UBUNTU_OPTS) do |n|
n.set["openstack"]["mq"] = {
"user" => "rabbit-user",
"vhost" => "/test-vhost"
}
end
@chef_run.converge "openstack-ops-messaging::rabbit-server"
end
it "overrides default rabbit attributes" do
expect(@chef_run.node["rabbitmq"]["default_user"]).to eql "rabbit-user"
expect(@chef_run.node['rabbitmq']['default_pass']).to eql "rabbit-pass"
expect(@chef_run.node['rabbitmq']['erlang_cookie']).to eql "erlang-cookie"
expect(@chef_run.node['rabbitmq']['cluster']).to be_true
expect(@chef_run.node['rabbitmq']['cluster_disk_nodes']).to eql ["rabbit-user@host1", "rabbit-user@host2"]
end
it "includes rabbit recipes" do
expect(@chef_run).to include_recipe "rabbitmq"
expect(@chef_run).to include_recipe "rabbitmq::mgmt_console"
end
describe "lwrps" do
it "deletes guest user" do
resource = @chef_run.find_resource(
"rabbitmq_user",
"guest"
).to_hash
expect(resource).to include(
:action => [:delete]
)
end
it "adds vhost" do
resource = @chef_run.find_resource(
"rabbitmq_vhost",
"/test-vhost"
).to_hash
expect(resource).to include(
:action => [:add]
)
end
it "adds user" do
resource = @chef_run.find_resource(
"rabbitmq_user",
"rabbit-user"
).to_hash
expect(resource).to include(
:password => "rabbit-pass",
:action => [:add]
)
end
it "sets user permissions" do
resource = @chef_run.find_resource(
"rabbitmq_user",
"rabbit-user"
).to_hash
#expect(resource).to include(
#:vhost => "/test-vhost",
#:permissions => '.* .* .*',
#:action => [:set_permissions]
#)
end
end
describe "mnesia" do
before do
::File.stub(:exists?).and_call_original
opts = ::UBUNTU_OPTS.merge(:evaluate_guards => true)
@chef_run = ::ChefSpec::ChefRunner.new opts
@cmd = <<-EOH.gsub(/^\s+/, "")
service rabbitmq-server stop;
rm -rf mnesia/;
touch .reset_mnesia_database;
service rabbitmq-server start
EOH
@file = "/var/lib/rabbitmq/.reset_mnesia_database"
end
it "resets database" do
::File.should_receive(:exists?).
with(@file).
and_return(false)
@chef_run.converge "openstack-ops-messaging::rabbit-server"
expect(@chef_run).to execute_command(@cmd).with(
:cwd => "/var/lib/rabbitmq"
)
end
it "doesn't reset database when already did" do
::File.should_receive(:exists?).
with(@file).
and_return(true)
@chef_run.converge "openstack-ops-messaging::rabbit-server"
expect(@chef_run).not_to execute_command(@cmd)
end
end
end
end

14
spec/server_spec.rb Normal file
View File

@ -0,0 +1,14 @@
require_relative "spec_helper"
describe "openstack-ops-messaging::server" do
before { ops_messaging_stubs }
describe "ubuntu" do
it "uses proper messaging server recipe" do
chef_run = ::ChefSpec::ChefRunner.new ::UBUNTU_OPTS
chef_run.converge "openstack-ops-messaging::server"
expect(chef_run).to include_recipe "openstack-ops-messaging::rabbit-server"
end
end
end

28
spec/spec_helper.rb Normal file
View File

@ -0,0 +1,28 @@
require "chefspec"
::LOG_LEVEL = :fatal
::REDHAT_OPTS = {
:platform => "redhat",
:version => "6.3",
:log_level => ::LOG_LEVEL
}
::UBUNTU_OPTS = {
:platform => "ubuntu",
:version => "12.04",
:log_level => ::LOG_LEVEL
}
def ops_messaging_stubs
::Chef::Recipe.any_instance.stub(:search).
with(:node, "roles:os-ops-messaging AND environment:_default").
and_return [
{ 'hostname' => 'host1' },
{ 'hostname' => 'host2' }
]
::Chef::Recipe.any_instance.stub(:user_password).
#with("rabbit-user"). #TODO(retr0h): uncomment once attributes are in openstack-common.
and_return "rabbit-pass"
::Chef::Recipe.any_instance.stub(:service_password).
with("rabbit_cookie").
and_return "erlang-cookie"
end