PoC of new version vmware dvs plugin for Fuel 8.0

The new agent-based neutron ml2 driver makes deployment process more
granular and complicated. Add new task and  new manifests for setup
and configure agents.

Change-Id: Ibf264640718e21dd02f43e0a755ace994fee5693

deployment_scripts/puppet/manifests/agents.pp

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 #    Copyright 2015 Mirantis, Inc.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -15,26 +12,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from nailgun.db.sqlalchemy.models import *
-from nailgun.db import db
-from copy import deepcopy
+notice('MODULAR: vmware_dvs agent install')
 
+$vcenter     = hiera('vcenter', {})
+$vmware_dvs  = hiera_hash('fuel-plugin-vmware-dvs', {})
+$neutron     = hiera_hash('neutron_config', {})
+$agents      = get_agents_data($vcenter, $neutron, $vmware_dvs)
 
-def clear_restriction():
-    for release in db().query(Release).all():
-        release.wizard_metadata = deepcopy(release.wizard_metadata)
-        for value in release.wizard_metadata['Network']['manager']['values']:
-            try:
-                del value['restrictions']
-                db().commit()
-            except:
-                pass
-    return 0
-
-
-def main():
-    return clear_restriction()
-
-
-if __name__ == "__main__":
-    main()
+create_resources(vmware_dvs::agent, $agents)

deployment_scripts/puppet/manifests/site.pp

@@ -14,21 +14,18 @@
 
 notice('MODULAR: fuel-plugin-vmware-dvs')
 
-$vc_hash               = hiera('vcenter', {})
-$dvs_hash              = hiera_hash('fuel-plugin-vmware-dvs', {})
-$neutron_hash          = hiera_hash('neutron_config', {})
-$vsphere_hostname      = inline_template('<%= @vc_hash["computes"][0]["vc_host"] %>')
-$vsphere_login         = inline_template('<%= @vc_hash["computes"][0]["vc_user"] %>')
-$vsphere_password      = inline_template('<%= @vc_hash["computes"][0]["vc_password"] %>')
-$dvs_network_maps      = inline_template('<%= @dvs_hash["vmware_dvs_net_maps"] %>')
-$neutron_physnet       = inline_template('<%= @neutron_hash["predefined_networks"]["admin_internal_net"]["L2"]["physnet"] %>')
+$vcenter     = hiera('vcenter', {})
+$vmware_dvs  = hiera_hash('fuel-plugin-vmware-dvs', {})
+$neutron     = hiera_hash('neutron_config', {})
 
-class {'vmware_dvs':
-  vsphere_hostname      => $vsphere_hostname,
-  vsphere_login         => $vsphere_login,
-  vsphere_password      => $vsphere_password,
-  network_maps          => $dvs_network_maps,
-  neutron_physnet       => $neutron_physnet,
-  driver_name           => 'vmware_dvs',
-  neutron_url_timeout   => '3600',
+$agents      = get_agents_data($vcenter, $neutron, $vmware_dvs)
+$agents_arr  = values($agents)
+$vc_creds    = $agents_arr[1]
+
+class {'::vmware_dvs':
+  vsphere_hostname    => $vc_creds['vsphere_hostname'],
+  vsphere_login       => $vc_creds['vsphere_login'],
+  vsphere_password    => $vc_creds['vsphere_password'],
+  network_maps        => $vc_creds['network_maps'],
+  neutron_url_timeout => '3600',
 }

deployment_scripts/puppet/modules/vmware_dvs/files/dvs_neutron_agent.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2015 Cloudbase Solutions.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import neutron.plugins.ml2.drivers.mech_vmware_dvs.agentDVS.dvs_neutron_agent as agent_main
+
+
+def main():
+    agent_main.main()

deployment_scripts/puppet/modules/vmware_dvs/files/neutron-dvs-agent

@@ -0,0 +1,10 @@
+#!/usr/bin/python
+# PBR Generated from u'console_scripts'
+
+import sys
+
+from neutron.cmd.eventlet.plugins.dvs_neutron_agent import main
+
+
+if __name__ == "__main__":
+    sys.exit(main())

deployment_scripts/puppet/modules/vmware_dvs/lib/puppet/parser/functions/get_agents_data.rb

@@ -0,0 +1,23 @@
+module Puppet::Parser::Functions
+  newfunction(:get_agents_data, :type => :rvalue,
+              :doc => <<-EOS
+              Create parameters for the agent resource
+              EOS
+             ) do |args|
+    raise(Puppet::ParseError, 'Shoud have 3 arguments!') if args.size < 2 or args[0] == ""
+    vcenter = args[0]['computes']
+    physnet = args[1]["predefined_networks"]["admin_internal_net"]["L2"]["physnet"]
+    dvSwitch = args[2]["vmware_dvs_net_maps"]
+    agents = []
+    vcenter.each {|vc|
+      agent = {}
+      agent["host"] = vc["availability_zone_name"] + "-" + vc["service_name"]
+      agent["vsphere_hostname"] = vc["vc_host"]
+      agent["vsphere_login"] = vc["vc_user"]
+      agent["vsphere_password"] = vc["vc_password"]
+      agent["network_maps"] = physnet + ":" + dvSwitch
+      agents.push(agent)
+    }
+    Hash[agents.collect {|agent| [agent["host"], agent]}]
+  end
+end

deployment_scripts/puppet/modules/vmware_dvs/lib/puppet/parser/functions/get_network_maps.rb

@@ -1,16 +0,0 @@
-module Puppet::Parser::Functions
-  newfunction(:get_network_maps, :type => :rvalue,
-              :doc => <<-EOS
-              Construct properly network_maps string
-              EOS
-             ) do |args|
-    raise(Puppet::ParseError, 'No name of dvSwitch provided!') if args.size < 1 or args[0] == ""
-    maps = args[0]
-    physnet = args[1]
-    if maps.include? ':'
-      maps
-    else
-      physnet + ":" + maps
-    end
-  end
-end

deployment_scripts/puppet/modules/vmware_dvs/manifests/agent.pp

@@ -0,0 +1,47 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+define vmware_dvs::agent(
+  $host             = 'vcenter-servicename',
+  $vsphere_hostname = '192.168.0.1',
+  $vsphere_login    = 'administrator@vsphere.local',
+  $vsphere_password = 'StrongPassword!',
+  $network_maps     = 'physnet1:dvSwitch1',
+)
+{
+  $agent_config = "/etc/neutron/plugins/ml2/vmware_dvs-${host}.ini"
+  $agent_name   = "neutron-plugin-vmware-dvs-agent-${host}"
+  $agent_init   = "/etc/init/${agent_name}.conf"
+  $agent_log    = "/var/log/neutron/vmware-dvs-agent-${host}.log"
+
+  file {$agent_config:
+    ensure  => present,
+    content => template('vmware_dvs/agent_config.erb'),
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+  }
+
+  file {$agent_init:
+    ensure  => present,
+    content => template('vmware_dvs/agent_init.erb'),
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+  }->
+  exec {"start_${agent_name}":
+    path    => '/usr/sbin:/sbin:/usr/bin:/bin',
+    command => "service ${agent_name} restart",
+  }
+}

deployment_scripts/puppet/modules/vmware_dvs/manifests/init.pp

@@ -14,61 +14,57 @@
 
 # == Class: ::vmware_dvs
 #
-# edit /etc/neutron/neturon.conf and /etc/neutron/plugin.ini
+# install the vmware_dvs neutron ml2 driver and configure the neutron for it
 #
 # === Parameters
 #
 # [*vsphere_hostname*]
-#   (required) String. This is the name or ip of VMware vSphere server
+#   (required) String. This is a name or ip of VMware vSphere server.
 #
 # [*vsphere_login*]
-#   (required) String. This is the name of VMware vSphere user
+#   (required) String. This is a name of VMware vSphere user.
 #
 # [*vsphere_password*]
-#   (required) String. This is the password of VMware vSphere user
+#   (required) String. This is a password of VMware vSphere user.
 #
 # [*network_maps*]
-#   (required) String. This is a name of distributed vSwitch
-#
-# [*neutron_physnet*]
-#   (required) String. This is a name of physnet of neutron.
-#
-# [*driver_name*]
-#   (optional) String. This is the name of installed driver.
+#   (required) String. This is a name of DVS.
 #
 # [*neutron_url_timeout*]
 #   (optional) String. This is the timeout for neutron
 
-
 class vmware_dvs(
-  $vsphere_hostname,
-  $vsphere_login,
-  $vsphere_password,
-  $network_maps,
-  $neutron_physnet,
-  $driver_name         = 'vmware_dvs',
+  $vsphere_hostname    = '192.168.0.1',
+  $vsphere_login       = 'administrator@vsphere.loc',
+  $vsphere_password    = 'StrongPassword!',
+  $network_maps        = 'physnet2:dvSwitch1',
   $neutron_url_timeout = '3600',
 )
 {
-  $true_network_maps  = get_network_maps($network_maps, $neutron_physnet)
-
-  Exec { path => '/usr/bin:/usr/sbin:/bin:/sbin' }
-
-  package {['python-suds','python-mech-vmware-dvs']:
-    ensure => present,
-  }
-
+  $py_root = '/usr/lib/python2.7/dist-packages'
   neutron_config {
     'DEFAULT/notification_driver': value => 'messagingv2';
     'DEFAULT/notification_topics': value => 'notifications,vmware_dvs';
-  } ->
+    }->
+    neutron_plugin_ml2 {
+      'ml2_vmware/vsphere_hostname': value => $vsphere_hostname;
+      'ml2_vmware/vsphere_login':    value => $vsphere_login;
+      'ml2_vmware/vsphere_password': value => $vsphere_password;
+      'ml2_vmware/network_maps':     value => $network_maps;
+      } ->
+      package { ['python-suds','python-mech-vmware-dvs']:
+        ensure => present,
+        }->
+        file {"${py_root}/neutron/plugins/ml2/drivers/mech_vmware_dvs":
+          ensure => 'link',
+          target => '/usr/local/lib/python2.7/dist-packages/mech_vmware_dvs',
+        }
 
-  neutron_plugin_ml2 {
-    'ml2_vmware/vsphere_hostname': value => $vsphere_hostname;
-    'ml2_vmware/vsphere_login':    value => $vsphere_login;
-    'ml2_vmware/vsphere_password': value => $vsphere_password;
-    'ml2_vmware/network_maps':     value => $true_network_maps;
-  } ->
+  service { 'neutron-server':
+    ensure    => running,
+    enable    => true,
+    subscribe => File["${py_root}/neutron/plugins/ml2/drivers/mech_vmware_dvs"],
+  }
 
   file_line { 'neutron_timeout':
     path  => '/etc/haproxy/conf.d/085-neutron.cfg',
@@ -76,12 +72,6 @@ class vmware_dvs(
     after => 'listen neutron',
   }
 
-  service { 'neutron-server':
-    ensure    => running,
-    enable    => true,
-    subscribe => [[Package['python-suds','python-mech-vmware-dvs']]],
-  }
-
   service {'haproxy':
     ensure     => running,
     hasrestart => true,
@@ -91,14 +81,27 @@ class vmware_dvs(
 
   nova_config {'neutron/url_timeout': value => $neutron_url_timeout}
 
-  file {'/usr/lib/python2.7/dist-packages/nova.patch':
+  file {"${py_root}/nova.patch":
     source => 'puppet:///modules/vmware_dvs/nova.patch',
     notify => Exec['apply-nova-patch'],
   }
   exec {'apply-nova-patch':
-    path        => '/usr/bin:/usr/sbin:/bin',
-    command     => 'patch -d /usr/lib/python2.7/dist-packages -N -p1
-    < /usr/lib/python2.7/dist-packages/nova.patch',
+    path        => '/usr/bin:/usr/sbin:/bin:/sbin',
+    command     => "patch -d ${py_root} -N -p1 < ${py_root}/nova.patch",
     refreshonly => true,
   }
+
+  file {'dvs_neutron_agent.py':
+    path   => "${py_root}/neutron/cmd/eventlet/plugins/dvs_neutron_agent.py",
+    source => 'puppet:///modules/vmware_dvs/dvs_neutron_agent.py',
+  }
+
+  file {'neutron-dvs-agent':
+    path    => '/usr/local/bin/neutron-dvs-agent',
+    source  => 'puppet:///modules/vmware_dvs/neutron-dvs-agent',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    require => Package['python-mech-vmware-dvs'],
+  }
 }

deployment_scripts/puppet/modules/vmware_dvs/templates/agent_config.erb

@@ -0,0 +1,12 @@
+[DEFAULT]
+host=<%= @host %>
+
+[securitygroup]
+enable_security_group = True
+firewall_driver=mech_vmware_dvs.agentDVS.vCenter_firewall.vCenterFirewallDriver
+
+[ml2_vmware]
+vsphere_login=<%= @vsphere_login %>
+network_maps=<%= @network_maps %>
+vsphere_hostname=<%= @vsphere_hostname %>
+vsphere_password=<%= @vsphere_password %>

deployment_scripts/puppet/modules/vmware_dvs/templates/agent_init.erb

@@ -0,0 +1,38 @@
+description "Neutron vmware dvs plugin agent"
+author "Igor Gajsin <igajsin@mirantis.com>"
+
+start on runlevel [2345]
+stop on runlevel [016]
+
+chdir /var/run
+
+pre-start script
+        mkdir -p /var/run/neutron
+        chown neutron:root /var/run/neutron
+end script
+
+script
+        [ -r /usr/share/neutron-common/plugin_guess_func ] || exit 0
+        . /usr/share/neutron-common/plugin_guess_func
+
+        if ! [ -r /etc/neutron/neutron.conf ] ; then
+                echo "Cloud not read /etc/neutron/neutron.conf: exiting"
+                exit 0
+        fi
+        CURRENT_PLUGIN=`grep "^[ \t]*core_plugin[ \t]*=[ \t]*[._a-zA-Z0-9]*\$" /etc/neutron/neutron.conf | sed -e 's/^[ \t]*core_plugin[ \t]*=[ \t]*//'`
+        if [ -z "${CURRENT_PLUGIN}" ] ; then
+                echo "No core_plugin= value found: please set it and try again"
+                exit 0
+        fi
+
+        neutron_core_plugin_to_plugin_name ${CURRENT_PLUGIN}
+        neutron_plugin_ini_path ${NEUTRON_PLUGIN_NAME}
+        [ -r "$NEUTRON_PLUGIN_CONFIG" ] && CONF_ARG="--config-file $NEUTRON_PLUGIN_CONFIG --config-file <%= @agent_config %>"
+        [ -r /etc/default/openstack ] && . /etc/default/openstack
+        [ -r /etc/default/$UPSTART_JOB ] && . /etc/default/$UPSTART_JOB
+
+        [ "x$USE_SYSLOG" = "xyes" ] && DAEMON_ARGS="$DAEMON_ARGS --use-syslog"
+        [ "x$USE_LOGFILE" != "xno" ] && DAEMON_ARGS="$DAEMON_ARGS --log-file=<%= @agent_log %>"
+        exec start-stop-daemon --start --chuid neutron:neutron --exec /usr/local/bin/neutron-dvs-agent -- \
+                --config-file=/etc/neutron/neutron.conf $CONF_ARG $DAEMON_ARGS
+end script

deployment_tasks.yaml

@@ -27,6 +27,16 @@
     puppet_modules:  puppet/modules:/etc/puppet/modules
     timeout: 720
 
+- id: vmware-dvs-neutron-agent-install
+  groups: ['primary-controller','controller','compute-vmware']
+  require_for: [openstack-network-networks]
+  requires: [openstack-network-plugins-l2]
+  type: puppet
+  parameters:
+    puppet_manifest: puppet/manifests/agents.pp
+    puppet_modules:  puppet/modules:/etc/puppet/modules
+    timeout: 720
+
 - id: vmware-dvs-compute-vmware
   groups: ['compute-vmware']
   requires:  [top-role-compute-vmware]

post_install.sh

@@ -13,6 +13,5 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-/usr/bin/dockerctl shell nailgun python /var/www/nailgun/plugins/fuel-plugin-vmware-dvs-2.0/deployment_scripts/strike.py
 /usr/bin/dockerctl shell nailgun sed -i -e "/if common_attrs.get('use_vcenter', {}).get('value') is True and/,+5 d" /usr/lib/python2.7/site-packages/nailgun/api/v1/validators/cluster.py
 /usr/bin/dockerctl shell nailgun systemctl restart nailgun.service