Refactoring provisioning part

Add first version of deploy part

bin/astute

@@ -73,9 +73,8 @@ end
 reporter = ConsoleReporter.new
 Astute.logger = Logger.new(STDOUT) if opts[:verbose]
 
-
 begin
-  environment = Astute::Cli::Enviroment.new(opts[:filename])
+  environment = Astute::Cli::Enviroment.new(opts[:filename], opts[:command])
 rescue Errno::ENOENT, Psych::SyntaxError, Astute::Cli::Enviroment::ValidationError => e
   report_and_exit(e, opts[:verbose])
 end
@@ -92,7 +91,7 @@ if environment['attributes'] && environment['attributes']['deployment_engine']
 end
 
 if [:deploy, :provision, :provision_and_deploy].include? opts[:command]
-  orchestrator = Astute::Orchestrator.new(deploy_engine, log_parsing=false)
+  orchestrator = Astute::Orchestrator.new(deploy_engine, log_parsing=true)
 end
 
 def console_provision(orchestrator, reporter, environment)

lib/astute/cli/deploy_schema.yaml

@@ -0,0 +1,291 @@
+type: map
+mapping:
+  "task_uuid":
+    type: text
+  "nodes":
+    type: seq
+    required: true
+    desc: Array of nodes
+    sequence:
+      - type: map
+        mapping:
+          "id":
+            type: int
+            unique: yes
+          "uid":
+            type: int
+            unique: yes
+          "fqdn":
+            type: text
+            desc: Fully-qualified domain name of the node
+          "role":
+            type: text
+            required: true
+            enum: ["primary-controller", "controller", "storage", "swift-proxy", "primary-swift-proxy"]
+          # Quantum true
+          "public_br":
+            type: text
+            desc: Name of the public bridge for Quantum-enabled configuration
+          # Quantum true
+          "internal_br":
+            type: text
+            desc: Name of the internal bridge for Quantum-enabled configuration
+          "network_data":
+            type: seq
+            desc: Array of network interfaces hashes
+            sequence:
+              - type: map
+                mapping:
+                  "name":
+                    type: text
+                    unique: true
+                    enum: ['management', 'public', 'storage', 'fixed']
+                    desc: Network type
+                  "dev":
+                    type: text
+                  "ip":
+                    type: text
+                  "netmask":
+                    type: text
+                  "gateway":
+                    type: text
+                
+  "attributes":
+    type: map
+    required: true
+    desc: General parameters for deployment
+    mapping:
+      "deployment_id":
+        type: int
+        desc: Id of deployment used do differentiate environments
+      "deployment_source":
+        type: text
+        enum: ['cli', 'web']
+        required: true
+      "management_vip":
+        type: text
+        required: true
+        desc: "Virtual IP address for internal services (MySQL, AMQP, internal OpenStack endpoints)"
+      "public_vip":
+        type: text
+        required: true
+        desc: "Virtual IP address for public services: Horizon, public OpenStack endpoints"
+      "master_ip":
+        type: text
+        required: true
+        desc: IP of puppet master
+      "deployment_mode":
+        type: text
+        enum: ['ha', 'ha_full', 'multinode']
+        desc:
+        required: true
+      "access":
+        type: map
+        required: true
+        mapping:
+          "password":
+            type: text
+            required: true
+          "user":
+            type: text
+            required: true
+          "tenant":
+            type: text
+            required: true
+          "email":
+            type: text
+            required: true
+      "use_cow_images":
+        type: bool
+        required: true
+        desc: Whether to use cow images
+      "auto_assign_floating_ip":
+        type: bool
+        required: true
+        desc: Whether to assign floating IPs automatically
+      "libvirt_type":
+        type: text
+        enum: [qemu, kvm]
+        required: true
+        desc: "Nova libvirt hypervisor type. Values: qemu|kvm"
+      "start_guests_on_host_boot":
+        type: bool
+        required: true
+      "quantum": 
+        type: bool
+        required: true
+      # Quantum true
+      "quantum_parameters":
+        type: map
+        mapping:
+          "tenant_network_type":
+            type: text
+            enum: ['gre', 'vlan']
+            required: true
+            desc: "Which type of network segmentation to use. Values: gre|vlan"
+          "segment_range":
+            type: text
+            required: true
+            desc: "Range of IDs for network segmentation. Consult Quantum documentation."
+          "metadata_proxy_shared_secret":
+            type: text
+            required: true
+            desc: Shared secret for metadata proxy services
+      "mysql":
+        type: map
+        required: true
+        desc: Credentials for MySQL
+        mapping:
+          "root_password":
+            type: text
+            required: true
+      "swift":
+        type: map
+        required: true
+        desc: Credentials for Swift
+        mapping:
+          "user_password":
+            type: text
+            required: true
+      "glance":
+        type: map
+        required: true
+        desc: Credentials for Glance
+        mapping:
+          "user_password":
+            type: text
+            required: true
+          "db_password":
+            type: text
+            required: true
+      "nova":
+        type: map
+        required: true
+        desc: Credentials for Nova
+        mapping:
+          "user_password":
+            type: text
+            required: true
+          "db_password":
+            type: text
+            required: true
+      "keystone":
+        type: map
+        required: true
+        desc: Credentials for Keystone
+        mapping:
+          "db_password":
+            type: text
+            required: true
+          "admin_token":
+            type: text
+            required: true
+      "quantum_access":
+        type: map
+        required: true
+        desc: Credentials for Quantum Access
+        mapping:
+          "user_password":
+            type: text
+            required: true
+          "db_password":
+            type: text
+            required: true
+      "rabbit":
+        type: map
+        required: true
+        desc: Credentials for RabbitMQ
+        mapping:
+          "user":
+            type: text
+            required: true
+          "password":
+            type: text
+            required: true
+      "cinder":
+        type: map
+        required: true
+        desc: Credentials for Cinder
+        mapping:
+          "user":
+            type: text
+            required: true
+          "password":
+            type: text
+            required: true
+      # Quantum false
+      "floating_network_range":
+        type: any
+        desc: Used for creation of floating networks/IPs during deployment (for quantum == false)
+      # Quantum true
+      "fixed_network_range":
+        type: any
+        desc: CIDR for fixed network created during deployment (for quantum == true)
+      "ntp_servers":
+        type: seq
+        required: true
+        desc: Array of ntp servers
+        sequence:
+          - type: "text"
+            required: true
+      "dns_nameservers":
+        type: seq
+        required: true
+        desc: Array of DNS servers configured during deployment phase
+        sequence:
+          - type: "text"
+            required: true
+      "cinder_nodes":
+        type: seq
+        desc: | 
+          Array of nodes to use as cinder-volume backends. Values: 'all'|<hostname>|
+          <internal IP address of node>|'controller'|<node_role>"
+        sequence:
+          - type: "text"
+            required: true
+      "base_syslog":
+        type: map
+        required: true
+        desc: Main syslog server configuration
+        mapping:
+          "syslog_server":
+            type: text
+            required: true
+          "syslog_port":
+            type: text
+            required: true
+      "syslog":
+        type: map
+        required: true
+        desc: Additional syslog servers configuration
+        mapping:
+          "syslog_port":
+            type: text
+          "syslog_transport":
+            type: text
+            enum: ['tcp', 'udp']
+          "syslog_server":
+            type: text
+      "horizon_use_ssl":
+        type: text
+        enum: ['false', 'default', 'exist', 'custom']
+        desc: Use HTTP or HTTPS for OpenStack dashboard (Horizon)
+      "use_unicast_corosync":
+        type: bool
+        default: false
+        desc: | 
+          Fuel uses Corosync and Pacemaker cluster engines for HA scenarios, thus 
+          requiring consistent multicast networking. Sometimes it is not possible to configure 
+          multicast in your network. In this case, you can tweak Corosync to use unicast addressing 
+          by setting use_unicast_corosync variable to true.
+      "auth_key":
+        type: text
+      "network_manager":
+        type: text  
+      "verbose":
+        type: bool
+        desc: How much information OpenStack provides when performing configuration (verbose mode)
+      "debug":
+        type: bool
+        desc: How much information OpenStack provides when performing configuration (debug mode)
+        

lib/astute/cli/enviroment.rb

@@ -32,10 +32,9 @@ module Astute
                                'mco_auto_setup', 'auth_key', 'puppet_version', 'mco_connector', 'mco_host']
       PROVISIONING_NET_KEYS = ['ip', 'power_address', 'mac', 'fqdn']
       
-      def initialize(file)
+      def initialize(file, operation)
         @config = YAML.load_file(file)
-        response = RestClient.get 'http://localhost:8000/api/nodes'
+        validate_enviroment(operation)
-        @api_data = JSON.parse(response).freeze
         to_full_config
       end
       
@@ -46,8 +45,6 @@ module Astute
       private
       
       def to_full_config
-        validate_enviroment
-        
         # Provision section
         @config['nodes'].each do |node|
           define_provisioning_network(node)
@@ -57,13 +54,14 @@ module Astute
           define_power_info(node)
           define_ks_meta(node)
           define_node_settings(node)
+          define_disks_section(node)
         end
         # Deploy section
       end
       
       
-      def validate_enviroment
+      def validate_enviroment(operation)
-        validator = YamlValidator.new
+        validator = YamlValidator.new(operation)
         errors = validator.validate(@config)
       
         errors.each do |e|
@@ -76,15 +74,22 @@ module Astute
         end
       end
       
-      # Set for node uniq id and uid from Nailgun
+      # Get data about discovered nodes using FuelWeb API 
-      def define_id_and_uid(node)
+      def find_node_api_data(node)
-        begin
+        @api_data ||= begin
-          id = @api_data.find{ |n| n['mac'].upcase == node['mac'].upcase }['id']
+          response = RestClient.get 'http://localhost:8000/api/nodes'
-        rescue
+          @api_data = JSON.parse(response).freeze
+        end
+        @api_data.find{ |n| n['mac'].upcase == node['mac'].upcase }
+        return @api_data if @api_data
         raise Enviroment::ValidationError, "Node #{node['name']} with mac adress #{node['mac']}
                                             not find among discovered nodes"
       end
       
+      # Set uniq id and uid for node from Nailgun using FuelWeb API
+      def define_id_and_uid(node)
+        id = find_node_api_data(node)['id']
+        
         # This params set for node by Nailgun and should not be edit by user
         node.merge!(
           'id'  => id,
@@ -92,6 +97,23 @@ module Astute
         )
       end
       
+      
+      # Set meta/disks section for node. This data used in provision to calculate the percentage 
+      # completion of the installation process.
+      # Example result for node['meta']
+      # "disks": [
+      #   {
+      #     "model": "VBOX HARDDISK", 
+      #     "disk": "disk/by-path/pci-0000:00:0d.0-scsi-0:0:0:0", 
+      #     "name": "sda", 
+      #     "size": 17179869184
+      #   }...
+      # ]
+      def define_disks_section(node)
+        node['meta'] ||= {}
+        node['meta']['disks'] = find_node_api_data(node)['meta']['disks']
+      end
+      
       def define_parameters(node, config_group_name, keys, position=nil)
         position ||= node
         if @config[config_group_name]
@@ -130,7 +152,7 @@ module Astute
         
         if provision_eth
           if provision_eth.absent?('ip_address')
-            api_node = @api_data.find{ |n| n['mac'].upcase == provision_eth['mac_address'].upcase }
+            api_node = find_node_api_data(node)
             api_provision_eth = api_node['meta']['interfaces'].find { |n| n['mac'].upcase == provision_eth['mac_address'].upcase }
             provision_eth['ip_address'] = api_provision_eth['ip'] 
             provision_eth['netmask'] = api_provision_eth['netmask']
@@ -150,8 +172,8 @@ module Astute
       
         absent_keys = node.absent_keys(PROVISIONING_NET_KEYS)
         if !absent_keys.empty?
-          raise "Please set 'use_for_provision' parameter for #{node['name']}
+          raise Enviroment::ValidationError, "Please set 'use_for_provision' parameter 
-                 or set manually #{absent_keys.each {|k| p k}}"
+                for #{node['name']} or set manually #{absent_keys.each {|k| p k}}"
         end
       end
     
@@ -160,7 +182,7 @@ module Astute
       #   eth0:
       #     ip_address: 10.20.0.188
       #     netmask: 255.255.255.0
-      #     dns_name: *fqdn
+      #     dns_name: controller-22.domain.tld
       #     static: '0'
       #     mac_address: 08:00:27:C2:06:DE
       # interfaces_extra:
@@ -209,7 +231,8 @@ module Astute
         end
     
         if node['ks_meta'].absent? 'ks_disks'
-          raise "Please set 'ks_disks' or 'ks_spaces' parameter in section ks_meta for #{node['name']}"
+          raise Enviroment::ValidationError, "Please set 'ks_disks' or 'ks_spaces' parameter 
+                in section ks_meta for #{node['name']}"
         end
     
         node['ks_meta']['ks_spaces'] = '"' + node['ks_meta']['ks_disks'].to_json.gsub("\"", "\\\"") + '"'

lib/astute/cli/provision_schema.yaml

@@ -5,6 +5,7 @@ mapping:
   "engine":
     type: map
     required: true
+    desc: Cobbler engine credentials
     mapping:
       "url":
         type: text
@@ -21,15 +22,20 @@ mapping:
       "power_type":
         type: text
         required: true
+        desc: Cobbler power-type. Consult cobbler documentation for available options.
       "power_user":
         type: text
         required: true
+        desc: Username for cobbler to manage power of this machine
       "power_pass":
         type: text
         required: true
+        desc: Password/credentials for cobbler to manage power of this machine
       "netboot_enabled":
         type: text
         required: true
+        desc: Disable/enable netboot for this node.
+        enum: ['0', '1']
   "common_node_settings":
     type: map
     mapping:
@@ -88,47 +94,60 @@ mapping:
   "nodes":
     type: seq
     required: true
+    desc: Array of nodes
     sequence:
       - type: map
         mapping:
           "id":
             type: int
             unique: yes
+            desc: MCollective node id in mcollective server.cfg
           "uid":
             type: int
             unique: yes
+            desc: UID of the node for deployment engine. Should be equal to `id`
           "name":
             type: text
             required: true
             unique: yes
+            desc: Name of the system in cobbler
           "hostname":
             type: text
             required: true
           "fqdn":
             type: text
+            desc: Fully-qualified domain name of the node
           "profile":
             type: text
             required: true
-            enum: ["centos-x86_64", "ubuntu_1204_x86_64"]
+            enum: ["centos-x86_64", "ubuntu_1204_x86_64", 'rhel-x86_64']
+            desc: Cobbler profile for the node.
           "ip":
             type: text
           "mac":
             type: text
           "power_address":
             type: text
+            desc: IP address of the device managing the node power state
           "power_type":
             type: text
+            desc: Cobbler power-type. Consult cobbler documentation for available options.
           "power_user":
             type: text
+            desc: Username for cobbler to manage power of this machine
           "name_servers":
             type: text
           "power_pass":
             type: text
+            desc: Password/credentials for cobbler to manage power of this machine
           "netboot_enabled":
             type: text
+            desc: Disable/enable netboot for this node.
+            enum: ['0', '1']
           "ks_meta":
             type: map
             required: true
+            desc: Kickstart metadata used during provisioning
             mapping:
               "mco_enable":
                 type: int

lib/astute/cli/yaml_validator.rb

@@ -18,10 +18,24 @@ module Astute
   module Cli
     class YamlValidator < Kwalify::Validator
     
-      def initialize
+      def initialize(operation)
-        gem_path =  Gem.loaded_specs['astute'].full_gem_path
+        schemas = if [:deploy, :provision].include? operation
-        schema_path = File.join(gem_path, 'lib', 'astute', 'cli', 'schema.yaml')
+          [operation]
-        @schema = Kwalify::Yaml.load_file(schema_path)
+        elsif operation == :provision_and_deploy
+          [:provision, :deploy]
+        else
+          raise "Incorrect scheme for validation"
+        end
+        
+        schema_hashes = []
+        schema_dir_path = File.expand_path(File.dirname(__FILE__)) 
+        schemas.each do |schema_name|
+          schema_path = File.join(schema_dir_path, "#{schema_name}_schema.yaml")
+          schema_hashes << YAML.load_file(schema_path)
+        end
+        
+        #FIXME: key 'hostname:' is undefined for provision_and_deploy. Why?
+        @schema = schema_hashes.size == 1 ? schema_hashes.first : schema_hashes[0].deep_merge(schema_hashes[1])
         super(@schema)
       end
     

lib/astute/ext/hash.rb

@@ -36,4 +36,12 @@ class Hash
     !absent?(key)
   end
   
+  def deep_merge(other_hash)
+    self.merge(other_hash) do |key, oldval, newval|
+      oldval = oldval.to_hash if oldval.respond_to?(:to_hash)
+      newval = newval.to_hash if newval.respond_to?(:to_hash)
+      oldval.class.to_s == 'Hash' && newval.class.to_s == 'Hash' ? oldval.deep_merge(newval) : newval
+    end
+  end
+   
 end