Split configuration astute.yaml into common and node parts

This file is huge if there are many nodes in cluster. Now
we generate it and store in memory for each node and it
takes a lot of time and memory.

Let's split it into common and node-specific parts so that
we can generate common part once and copy it from master
node to each node affected by deployment.

Change-Id: I114a8598460bef9063e43505f46fe38cd564d8c7
Closes-Bug: #1660308

deployment/puppet/openstack_tasks/examples/keystone/tasks.yaml

@@ -150,7 +150,7 @@
   condition:
     yaql_exp: &changed_username >
       changed($.access.user)
-  requires: [upload_configuration]
+  requires: [copy_cluster_configuration]
   required_for: [pre_deployment_end]
   parameters:
     path: /etc/fuel/cluster/{CLUSTER_ID}/old_admin_user.yaml

deployment/puppet/osnailyfacter/manifests/hiera/hiera.pp

@@ -5,7 +5,7 @@ class osnailyfacter::hiera::hiera {
   $data_dir            = '/etc/hiera'
   $override_dir        = 'plugins'
   $override_dir_path   = "${data_dir}/${override_dir}"
-  $metadata_file       = '/etc/astute.yaml'
+  $metadata_file       = '/etc/hiera/cluster.yaml'
 
   $data = [
     'override/node/%{::fqdn}%{disable_globals_yaml}',
@@ -24,13 +24,12 @@ class osnailyfacter::hiera::hiera {
     'nodes%{disable_globals_yaml}',
     'old_admin_user%{disable_globals_yaml}',
     'globals%{disable_globals_yaml}',
-    'astute',
+    'node',
+    'cluster',
   ]
 
-  $astute_data_file    = '/etc/astute.yaml'
   $hiera_main_config   = '/etc/hiera.yaml'
   $hiera_puppet_config = '/etc/puppet/hiera.yaml'
-  $hiera_data_file     = "${data_dir}/astute.yaml"
 
   File {
     owner => 'root',
@@ -77,12 +76,6 @@ class osnailyfacter::hiera::hiera {
     mode   => '0640',
   }
 
-  file { 'hiera_data_astute' :
-    ensure => 'symlink',
-    path   => $hiera_data_file,
-    target => $astute_data_file,
-  }
-
   file { 'hiera_puppet_config' :
     ensure => 'symlink',
     path   => $hiera_puppet_config,

deployment/puppet/osnailyfacter/modular/astute/tasks.yaml

@@ -49,7 +49,7 @@
   role: ['/.*/']
   requires: [post_deployment_start]
 
-- id: upload_configuration
+- id: upload_node_configuration
   type: upload_file
   version: 2.1.0
   role: ['master', '/.*/']
@@ -59,25 +59,87 @@
   required_for: [pre_deployment_start]
   refresh_on: ['*']
   parameters:
-    path: /etc/fuel/cluster/{CLUSTER_ID}/astute.yaml
+    path: /etc/fuel/cluster/{CLUSTER_ID}/node.yaml
     permissions: '0640'
     dir_permissions: '0750'
     timeout: 180
     data:
-      yaql_exp: '$.toYaml()'
+      yaql_exp: '$node.toYaml()'
 
-- id: configuration_symlink
+- id: upload_cluster_configuration
+  type: upload_file
+  version: 2.1.0
+  role: ['master']
+  condition:
+    yaql_exp: 'changed($)'
+  requires: []
+  required_for: [copy_cluster_configuration]
+  refresh_on: ['*']
+  parameters:
+    path: /var/lib/fuel/configs/{CLUSTER_ID}/cluster.yaml
+    permissions: '0640'
+    dir_permissions: '0750'
+    timeout: 180
+    data:
+      yaql_exp: '$common.toYaml()'
+
+- id: copy_cluster_configuration
+  type: copy_files
+  version: 2.1.0
+  role: ['master', '/.*/']
+  condition:
+    yaql_exp: 'changed($)'
+  required_for: [pre_deployment_start]
+  requires: [upload_cluster_configuration]
+  cross-depends:
+      - name: upload_cluster_configuration
+        role: master
+  parameters:
+    files:
+      - src: /var/lib/fuel/configs/{CLUSTER_ID}/cluster.yaml
+        dst: /etc/fuel/cluster/{CLUSTER_ID}/cluster.yaml
+    permissions: '0644'
+    dir_permissions: '0755'
+
+- id: configuration_symlink_cluster
   type: shell
   version: 2.1.0
   role: ['/.*/']
   condition:
     yaql_exp: '$.uid in added($.network_metadata.nodes.values()).uid'
-  requires: [upload_configuration]
+  requires: [copy_cluster_configuration]
   required_for: [pre_deployment_start]
   parameters:
-    cmd: ln -sf /etc/fuel/cluster/{CLUSTER_ID}/astute.yaml /etc/astute.yaml
+    cmd: ln -sf /etc/fuel/cluster/{CLUSTER_ID}/cluster.yaml /etc/hiera/cluster.yaml
     timeout: 180
 
+- id: configuration_symlink_node
+  type: shell
+  version: 2.1.0
+  role: ['/.*/']
+  condition:
+    yaql_exp: '$.uid in added($.nodes).uid'
+  requires: [upload_node_configuration]
+  required_for: [pre_deployment_start]
+  parameters:
+    cmd: ln -sf /etc/fuel/cluster/{CLUSTER_ID}/node.yaml /etc/hiera/node.yaml
+    timeout: 180
+
+- id: upload_deprecated_astute_conf
+  type: upload_file
+  version: 2.1.0
+  role: ['master', '/.*/']
+  condition:
+    yaql_exp: 'changed($)'
+  requires: []
+  required_for: [copy_cluster_configuration]
+  refresh_on: ['*']
+  parameters:
+    path: /etc/astute.yaml
+    permissions: '0644'
+    timeout: 180
+    data: "# This file is deprecated and deployment info can be got via hiera."
+
 - id: update_hosts
   type: puppet
   version: 2.1.0
@@ -401,7 +463,7 @@
   role: master
   condition:
     yaql_exp: &deleted_nodes 'changed($.network_metadata.nodes)'
-  requires: [upload_configuration]
+  requires: [copy_cluster_configuration]
   required_for: [pre_deployment_end]
   parameters:
     path: /etc/fuel/cluster/{CLUSTER_ID}/deleted_nodes.yaml

deployment/puppet/osnailyfacter/modular/plugins/tasks.yaml

@@ -2,7 +2,7 @@
   type: puppet
   version: 2.1.0
   groups: ['/.*/']
-  requires: [configuration_symlink]
+  requires: [configuration_symlink_cluster]
   required_for: [pre_deployment_start]
   condition:
     yaql_exp: &have_plugins "len($.plugins) > 0"

tests/noop/spec/hosts/hiera/hiera_spec.rb

@@ -38,17 +38,11 @@ describe manifest do
       # ensure hiera_config is taking plugin overrides from the astute.yaml
       should contain_hiera_config('/etc/hiera.yaml').with(
         'ensure' => 'present',
-        'metadata_yaml_file' => '/etc/astute.yaml',
+        'metadata_yaml_file' => '/etc/hiera/cluster.yaml',
         'plugins_dir' => 'plugins',
         'data_dir' => '/etc/hiera',
       )
 
-      # check symlinks
-      should contain_file('hiera_data_astute').with(
-        'ensure' => 'symlink',
-        'path'   => '/etc/hiera/astute.yaml',
-        'target' => '/etc/astute.yaml'
-      )
       should contain_file('hiera_puppet_config').with(
         'ensure' => 'symlink',
         'path'   => '/etc/puppet/hiera.yaml',