Enhancements and fixes to various agents

* bugfixes to l3, dhcp, and openvswitch
* plugin lookup fixes for spec tests
* install kernel headers for openvswitch
* add spec tests for openvswitch
* add spec tests for dhcp_agent
* add spec skeleton tests for l3_agent

Change-Id: I68177c191e355dbfced23834a627cca5adf7abee

.gitignore

@@ -1,3 +1,4 @@
+Vagrantfile
 Berksfile.lock
 validation.pem
 metadata.json

attributes/default.rb

@@ -74,6 +74,12 @@ default["openstack"]["network"]["core_plugin"] = "quantum.plugins.openvswitch.ov
 
 default["openstack"]["network"]["interface_driver"] = 'quantum.agent.linux.interface.OVSInterfaceDriver'
 
+# maps the above driver to a plugin name
+default["openstack"]["network"]["interface_driver_map"] = {
+   'ovsinterfacedriver' => 'openvswitch',
+   'bridgeinterfacedriver' => 'linuxbridge'
+}
+
 # The agent can use other DHCP drivers.  Dnsmasq is the simplest and requires
 # no additional setup of the DHCP server.
 default["openstack"]["network"]["dhcp_driver"] = 'quantum.agent.linux.dhcp.Dnsmasq'
@@ -106,6 +112,29 @@ default["openstack"]["network"]["dhcp"]["enable_isolated_metadata"] = "False"
 # This option requires enable_isolated_metadata = True
 default["openstack"]["network"]["dhcp"]["enable_metadata_network"] = "False"
 
+# On ubuntu precise, we build dnsmasq from source to fetch a more recent
+# version of dnsmasq since a backport is not available. For any other
+# platform, dnsmasq will be installed as a package
+#
+# See https://lists.launchpad.net/openstack/msg11696.html
+default["openstack"]["network"]["dhcp"]["dnsmasq_url"] = "https://github.com/guns/dnsmasq/archive/v2.65.tar.gz"
+
+# The name of the file we will fetch
+default["openstack"]["network"]["dhcp"]["dnsmasq_filename"] = "v2.65.tar.gz"
+
+# The checksum of the remote file we fetched
+default["openstack"]["network"]["dhcp"]["dnsmasq_checksum"] = "f6cab8c64cb612089174f50927a05e2b"
+
+# The package architecture that will be built which should match the
+# archecture of the server this cookbook will run on which will be
+# amd64 or i386
+default["openstack"]["network"]["dhcp"]["dnsmasq_architecture"] = "amd64"
+
+# The debian package version that the above tarball will produce
+default["openstack"]["network"]["dhcp"]["dnsmasq_dpkgversion"] = "2.65-1"
+
+
+
 # ============================= L3 Agent Configuration =====================
 
 # If use_namespaces is set as False then the agent can only configure one router.
@@ -595,6 +624,7 @@ when "fedora", "redhat", "centos" # :pragma-foodcritic: ~FC024 - won't fix this
     "quantum_packages" => [ "openstack-quantum" ],
     "quantum_dhcp_packages" => [ "openstack-quantum" ],
     "quantum_metadata_agent_packages" => [ "quantum-metadata-agent" ],
+    "quantum_dhcp_build_packages" => [],
     "quantum_l3_packages" => [ "quantum-l3-agent" ],
     "quantum_plugin_package" => "openstack-quantum-%plugin%",
     "quantum_server_service" => "quantum-server",
@@ -612,6 +642,7 @@ when "suse"
     "nova_network_packages" => ["openstack-nova-network"],
     "quantum_packages" => ["openstack-quantum"],
     "quantum_dhcp_packages" => ["openstack-quantum-dhcp-agent"],
+    "quantum_dhcp_build_packages" => [],
     "quantum_l3_packages" => ["openstack-quantum-l3-agent"],
     "quantum_plugin_package" => "openstack-quantum-%plugin%",
     "quantum_openvswitch_packages" => ["openstack-quantum-openvswitch-agent"],
@@ -631,6 +662,7 @@ when "ubuntu"
     "nova_network_packages" => [ "nova-network" ],
     "quantum_packages" => [ "quantum-server", "python-quantumclient", "python-pyparsing", "python-cliff" ],
     "quantum_dhcp_packages" => [ "quantum-dhcp-agent" ],
+    "quantum_dhcp_build_packages" => [ "build-essential", "pkg-config", "libidn11-dev", "libdbus-1-dev", "libnetfilter-conntrack-dev" ],
     "quantum_l3_packages" => [ "quantum-l3-agent" ],
     "quantum_plugin_package" => "quantum-plugin-%plugin%",
     "quantum_openvswitch_packages" => [ "openvswitch-switch", "openvswitch-datapath-dkms", "bridge-utils" ],

recipes/dhcp_agent.rb

@@ -17,6 +17,10 @@
 # limitations under the License.
 #
 
+platform_options = node["openstack"]["network"]["platform"]
+driver_name = node["openstack"]["network"]["interface_driver"].split('.').last.downcase
+main_plugin = node["openstack"]["network"]["interface_driver_map"][driver_name]
+
 # This will copy recursively all the files in
 # /files/default/etc/quantum/rootwrap.d
 remote_directory "/etc/quantum/rootwrap.d" do
@@ -46,8 +50,6 @@ end
 
 # Some plugins have DHCP functionality, so we install the plugin
 # Python package and include the plugin-specific recipe here...
-main_plugin = node["openstack"]["network"]["interface_driver"].split('.').last.downcase
-
 package platform_options["quantum_plugin_package"].gsub("%plugin%", main_plugin) do
   action :install
 end
@@ -63,6 +65,68 @@ template "/etc/quantum/dhcp_agent.ini" do
   owner node["openstack"]["network"]["platform"]["user"]
   group node["openstack"]["network"]["platform"]["group"]
   mode   00644
-
   notifies :restart, "service[quantum-dhcp-agent]", :immediately
 end
+
+# Deal with ubuntu precise dnsmasq 2.59 version by custom
+# compiling a more recent version of dnsmasq
+#
+# See:
+# https://lists.launchpad.net/openstack/msg11696.html
+# https://bugs.launchpad.net/ubuntu/+source/dnsmasq/+bug/1013529
+# https://bugs.launchpad.net/ubuntu/+source/dnsmasq/+bug/1103357
+# http://www.thekelleys.org.uk/dnsmasq/CHANGELOG (SO_BINDTODEVICE)
+#
+# Would prefer a PPA or backport but there are none and upstream
+# has no plans to fix
+if node['lsb']['codename'] == "precise"
+
+  platform_options["quantum_dhcp_build_packages"].each do |pkg|
+    package pkg do
+      action :install
+    end
+  end
+
+  dhcp_options = node['openstack']['network']['dhcp']
+
+  src_filename = dhcp_options['dnsmasq_filename']
+  src_filepath = "#{Chef::Config['file_cache_path']}/#{src_filename}"
+  extract_path = "#{Chef::Config['file_cache_path']}/#{dhcp_options['dnsmasq_checksum']}"
+
+  remote_file src_filepath do
+    source dhcp_options['dnsmasq_url']
+    checksum dhcp_options['dnsmasq_checksum']
+    owner 'root'
+    group 'root'
+    mode 00644
+  end
+
+  bash 'extract_package' do
+    cwd ::File.dirname(src_filepath)
+    code <<-EOH
+      mkdir -p #{extract_path}
+      tar xzf #{src_filename} -C #{extract_path}
+      mv #{extract_path}/*/* #{extract_path}/
+      cd #{extract_path}/
+      debian/rules binary
+      EOH
+    not_if { ::File.exists?(extract_path) }
+    notifies :install, "dpkg_package[dnsmasq]", :immediately
+    notifies :install, "dpkg_package[dnsmasq-utils]", :immediately
+    notifies :install, "dpkg_package[dnsmasq-base]", :immediately
+  end
+
+  dpkg_package "dnsmasq" do
+    source "#{extract_path}/../dnsmasq_#{dhcp_options['dnsmasq_dpkgversion']}_all.deb"
+    action :nothing
+  end
+  dpkg_package "dnsmasq-utils" do
+    source "#{extract_path}/../dnsmasq-utils_#{dhcp_options['dnsmasq_dpkgversion']}_#{dhcp_options['dnsmasq_architecture']}.deb"
+    action :nothing
+  end
+  dpkg_package "dnsmasq-base" do
+    source "#{extract_path}/../dnsmasq-base_#{dhcp_options['dnsmasq_dpkgversion']}_#{dhcp_options['dnsmasq_architecture']}.deb"
+    action :nothing
+  end
+
+end

recipes/l3_agent.rb

@@ -17,9 +17,9 @@
 # limitations under the License.
 #
 
-# Some plugins have L3 functionality, so we install the plugin
-# Python package and include the plugin-specific recipe here...
-main_plugin = node["openstack"]["network"]["interface_driver"].split('.').last.downcase
+platform_options = node["openstack"]["network"]["platform"]
+driver_name = node["openstack"]["network"]["interface_driver"].split('.').last.downcase
+main_plugin = node["openstack"]["network"]["interface_driver_map"][driver_name]
 
 # This will copy recursively all the files in
 # /files/default/etc/quantum/rootwrap.d
@@ -70,7 +70,6 @@ template "/etc/quantum/l3_agent.ini" do
   owner node["openstack"]["network"]["platform"]["user"]
   group node["openstack"]["network"]["platform"]["group"]
   mode   00644
-
   notifies :restart, "service[quantum-l3-agent]", :immediately
 end
 

recipes/openvswitch.rb

@@ -17,12 +17,25 @@
 # limitations under the License.
 #
 
+require 'uri'
+
+class ::Chef::Recipe
+  include ::Openstack
+end
+
+platform_options = node["openstack"]["network"]["platform"]
+
 # discover database attributes
 db_user = node["openstack"]["network"]["db"]["username"]
 db_pass = db_password "quantum"
 sql_connection = db_uri("network", db_user, db_pass)
 
-platform_options = node["openstack"]["network"]["platform"]
+bash "installing linux headers to compile openvswitch module" do
+  only_if { platform?(%w(ubuntu debian)) } # :pragma-foodcritic: ~FC024 - won't fix this
+  code <<-EOH
+    apt-get install -y linux-headers-`uname -r`
+  EOH
+end
 
 platform_options["quantum_openvswitch_packages"].each do |pkg|
   package pkg do
@@ -40,14 +53,15 @@ execute "quantum-node-setup --plugin openvswitch" do
   only_if { platform?(%w(fedora redhat centos)) } # :pragma-foodcritic: ~FC024 - won't fix this
 end
 
-template "/etc/quantum/plugins/openvswitch/ovs_quantum_plugin.ini" do
-  source "plugins/openvswitch/ovs_quantum_plugin.ini.erb"
-  owner node["openstack"]["network"]["platform"]["user"]
-  group node["openstack"]["network"]["platform"]["group"]
-  mode 00644
-  variables(
-    :sql_connection => sql_connection
-  )
-
-  notifies :restart, "service[quantum-server]", :immediately
+if node.run_list.expand(node.chef_environment).recipes.include?("openstack-network::server")
+  template "/etc/quantum/plugins/openvswitch/ovs_quantum_plugin.ini" do
+    source "plugins/openvswitch/ovs_quantum_plugin.ini.erb"
+    owner node["openstack"]["network"]["platform"]["user"]
+    group node["openstack"]["network"]["platform"]["group"]
+    mode 00644
+    variables(
+      :sql_connection => sql_connection
+    )
+    notifies :restart, "service[quantum-server]", :immediately
+  end
 end

recipes/server.rb

@@ -24,7 +24,9 @@ class ::Chef::Recipe
   include ::Openstack
 end
 
-main_plugin = node["openstack"]["network"]["plugins"].first.downcase
+platform_options = node["openstack"]["network"]["platform"]
+driver_name = node["openstack"]["network"]["interface_driver"].split('.').last.downcase
+main_plugin = node["openstack"]["network"]["interface_driver_map"][driver_name]
 core_plugin = node["openstack"]["network"]["core_plugin"]
 
 if node["openstack"]["network"]["syslog"]["use"]

spec/dhcp_agent_spec.rb

@@ -0,0 +1,58 @@
+require_relative 'spec_helper'
+
+describe 'openstack-network::dhcp_agent' do
+
+  describe "ubuntu" do
+
+    before do
+      quantum_stubs
+      @chef_run = ::ChefSpec::ChefRunner.new ::UBUNTU_OPTS
+      @chef_run.converge "openstack-network::dhcp_agent"
+    end
+
+    # since our mocked version of ubuntu is precise, our compile
+    # utilities should be installed to build dnsmasq
+    it "installs dnsmasq build dependencies" do
+      [ "build-essential", "pkg-config", "libidn11-dev", "libdbus-1-dev", "libnetfilter-conntrack-dev" ].each do |pkg|
+        expect(@chef_run).to install_package pkg
+      end
+    end
+
+    it "installs quamtum dhcp package" do
+      expect(@chef_run).to install_package "quantum-dhcp-agent"
+    end
+
+    it "starts the dhcp agent on boot" do
+     expect(@chef_run).to set_service_to_start_on_boot "quantum-dhcp-agent"
+    end
+
+    describe "/etc/quantum/plugins" do
+      before do
+        @file = @chef_run.directory "/etc/quantum/plugins"
+      end
+      it "has proper owner" do
+        expect(@file).to be_owned_by "quantum", "quantum"
+      end
+      it "has proper modes" do
+        expect(sprintf("%o", @file.mode)).to eq "700"
+      end
+    end
+
+    describe "/etc/quantum/dhcp_agent.ini" do
+      before do
+        @file = @chef_run.template "/etc/quantum/dhcp_agent.ini"
+      end
+      it "has proper owner" do
+        expect(@file).to be_owned_by "quantum", "quantum"
+      end
+      it "has proper modes" do
+        expect(sprintf("%o", @file.mode)).to eq "644"
+      end
+      it "template contents" do
+        pending "TODO: implement"
+      end
+    end
+
+  end
+
+end

spec/l3_agent_spec.rb

@@ -0,0 +1,19 @@
+require_relative 'spec_helper'
+
+describe 'openstack-network::l3_agent' do
+
+  describe "ubuntu" do
+
+    before do
+      quantum_stubs
+      @chef_run = ::ChefSpec::ChefRunner.new ::UBUNTU_OPTS
+      @chef_run.converge "openstack-network::l3_agent"
+    end
+
+    it "installs quamtum l3 package" do
+      expect(@chef_run).to install_package "quantum-l3-agent"
+    end
+
+  end
+
+end

spec/openvswitch_spec.rb

@@ -0,0 +1,31 @@
+require_relative 'spec_helper'
+
+describe 'openstack-network::openvswitch' do
+
+  describe "ubuntu" do
+
+    before do
+      quantum_stubs
+      @chef_run = ::ChefSpec::ChefRunner.new ::UBUNTU_OPTS
+      @chef_run.converge "openstack-network::openvswitch"
+    end
+
+    it "installs openvswitch switch" do
+      expect(@chef_run).to install_package "openvswitch-switch"
+    end
+    it "installs openvswitch datapath dkms" do
+      expect(@chef_run).to install_package "openvswitch-datapath-dkms"
+    end
+    it "installs linux bridge utils" do
+      expect(@chef_run).to install_package "bridge-utils"
+    end
+    it "installs linux kernel headers" do
+      expect(@chef_run).to execute_bash_script "installing linux headers to compile openvswitch module"
+    end
+    it "sets the openvswitch service to start on boot" do
+      expect(@chef_run).to set_service_to_start_on_boot 'openvswitch-switch'
+    end
+
+  end
+
+end

spec/server_spec.rb

@@ -7,11 +7,6 @@ describe 'openstack-network::server' do
     before do
       quantum_stubs
       @chef_run = ::ChefSpec::ChefRunner.new ::UBUNTU_OPTS
-      @node = @chef_run.node
-
-      # mock out an interface on the storage node
-      @node.set["network"] = MOCK_NODE_NETWORK_DATA['network']
-
       @chef_run.converge "openstack-network::server"
     end