Make removing nodes from scaled items possible.

This makes it possible to remove a dead node (e.g. if NovaCompute2 has
failed, regenerate the template with ,2 in the scale parameter, and
NovaCompute2 will not be enumerated.

Change-Id: I65d85a88152ed4adee60895173f8a05611a6440b

examples/scale_map_result_hot_blacklist.yaml

@@ -0,0 +1,367 @@
+description: examples/scale_map_hot.yaml
+heat_template_version: '2013-05-23'
+resources:
+  ComputeUser:
+    properties:
+      Policies:
+      - get_param: ComputeAccessPolicy
+    type: AWS::IAM::User
+  GlobalAccessPolicy:
+    type: OS::Heat::AccessPolicy
+  NovaCompute0:
+    metadata:
+      os-collect-config:
+        cfn:
+          access_key_id:
+            get_resource: NovaCompute0Key
+          path: NovaCompute0Config.Metadata
+          secret_access_key:
+            get_attr:
+            - NovaCompute0Key
+            - SecretAccessKey
+          stack_name:
+            get_param: AWS::StackName
+    properties:
+      image:
+        get_param: ComputeImage
+    type: OS::Nova::Server
+  NovaCompute0CompletionCondition:
+    depends_on: notcompute
+    properties:
+      Count: '1'
+      Handle:
+        get_resource: NovaCompute0CompletionHandle
+      Timeout: '1800'
+    type: AWS::CloudFormation::WaitCondition
+  NovaCompute0CompletionHandle:
+    type: AWS::CloudFormation::WaitConditionHandle
+  NovaCompute0Config:
+    metadata:
+      completion-handle:
+        get_resource: NovaCompute0CompletionHandle
+      hosts:
+        list_join:
+        - '
+
+          '
+        - - list_join:
+            - ' '
+            - - Fn::Select:
+                - 0
+                - Fn::Select:
+                  - ctlplane
+                  - get_attr:
+                    - NovaCompute0
+                    - networks
+              - Fn::Select:
+                - name
+                - get_attr:
+                  - NovaCompute0
+                  - show
+              - list_join:
+                - .
+                - - Fn::Select:
+                    - name
+                    - get_attr:
+                      - NovaCompute0
+                      - show
+                  - local
+          - list_join:
+            - ' '
+            - - Fn::Select:
+                - 0
+                - Fn::Select:
+                  - ctlplane
+                  - get_attr:
+                    - NovaCompute3
+                    - networks
+              - Fn::Select:
+                - name
+                - get_attr:
+                  - NovaCompute3
+                  - show
+              - list_join:
+                - .
+                - - Fn::Select:
+                    - name
+                    - get_attr:
+                      - NovaCompute3
+                      - show
+                  - local
+          - list_join:
+            - ' '
+            - - Fn::Select:
+                - 0
+                - Fn::Select:
+                  - ctlplane
+                  - get_attr:
+                    - NovaCompute4
+                    - networks
+              - Fn::Select:
+                - name
+                - get_attr:
+                  - NovaCompute4
+                  - show
+              - list_join:
+                - .
+                - - Fn::Select:
+                    - name
+                    - get_attr:
+                      - NovaCompute4
+                      - show
+                  - local
+      os-collect-config:
+        cfn:
+          access_key_id:
+            get_resource: NovaCompute0Key
+          path: NovaCompute0Config.Metadata
+          secret_access_key:
+            get_attr:
+            - NovaCompute0Key
+            - SecretAccessKey
+          stack_name:
+            get_param: AWS::StackName
+    type: AWS::AutoScaling::LaunchConfiguration
+  NovaCompute0Key:
+    properties:
+      UserName:
+        get_param: ComputeUser
+    type: AWS::IAM::AccessKey
+  NovaCompute3:
+    metadata:
+      os-collect-config:
+        cfn:
+          access_key_id:
+            get_resource: NovaCompute3Key
+          path: NovaCompute3Config.Metadata
+          secret_access_key:
+            get_attr:
+            - NovaCompute3Key
+            - SecretAccessKey
+          stack_name:
+            get_param: AWS::StackName
+    properties:
+      image:
+        get_param: ComputeImage
+    type: OS::Nova::Server
+  NovaCompute3CompletionCondition:
+    depends_on: notcompute
+    properties:
+      Count: '1'
+      Handle:
+        get_resource: NovaCompute3CompletionHandle
+      Timeout: '1800'
+    type: AWS::CloudFormation::WaitCondition
+  NovaCompute3CompletionHandle:
+    type: AWS::CloudFormation::WaitConditionHandle
+  NovaCompute3Config:
+    metadata:
+      completion-handle:
+        get_resource: NovaCompute3CompletionHandle
+      hosts:
+        list_join:
+        - '
+
+          '
+        - - list_join:
+            - ' '
+            - - Fn::Select:
+                - 0
+                - Fn::Select:
+                  - ctlplane
+                  - get_attr:
+                    - NovaCompute0
+                    - networks
+              - Fn::Select:
+                - name
+                - get_attr:
+                  - NovaCompute0
+                  - show
+              - list_join:
+                - .
+                - - Fn::Select:
+                    - name
+                    - get_attr:
+                      - NovaCompute0
+                      - show
+                  - local
+          - list_join:
+            - ' '
+            - - Fn::Select:
+                - 0
+                - Fn::Select:
+                  - ctlplane
+                  - get_attr:
+                    - NovaCompute3
+                    - networks
+              - Fn::Select:
+                - name
+                - get_attr:
+                  - NovaCompute3
+                  - show
+              - list_join:
+                - .
+                - - Fn::Select:
+                    - name
+                    - get_attr:
+                      - NovaCompute3
+                      - show
+                  - local
+          - list_join:
+            - ' '
+            - - Fn::Select:
+                - 0
+                - Fn::Select:
+                  - ctlplane
+                  - get_attr:
+                    - NovaCompute4
+                    - networks
+              - Fn::Select:
+                - name
+                - get_attr:
+                  - NovaCompute4
+                  - show
+              - list_join:
+                - .
+                - - Fn::Select:
+                    - name
+                    - get_attr:
+                      - NovaCompute4
+                      - show
+                  - local
+      os-collect-config:
+        cfn:
+          access_key_id:
+            get_resource: NovaCompute3Key
+          path: NovaCompute3Config.Metadata
+          secret_access_key:
+            get_attr:
+            - NovaCompute3Key
+            - SecretAccessKey
+          stack_name:
+            get_param: AWS::StackName
+    type: AWS::AutoScaling::LaunchConfiguration
+  NovaCompute3Key:
+    properties:
+      UserName:
+        get_param: ComputeUser
+    type: AWS::IAM::AccessKey
+  NovaCompute4:
+    metadata:
+      os-collect-config:
+        cfn:
+          access_key_id:
+            get_resource: NovaCompute4Key
+          path: NovaCompute4Config.Metadata
+          secret_access_key:
+            get_attr:
+            - NovaCompute4Key
+            - SecretAccessKey
+          stack_name:
+            get_param: AWS::StackName
+    properties:
+      image:
+        get_param: ComputeImage
+    type: OS::Nova::Server
+  NovaCompute4CompletionCondition:
+    depends_on: notcompute
+    properties:
+      Count: '1'
+      Handle:
+        get_resource: NovaCompute4CompletionHandle
+      Timeout: '1800'
+    type: AWS::CloudFormation::WaitCondition
+  NovaCompute4CompletionHandle:
+    type: AWS::CloudFormation::WaitConditionHandle
+  NovaCompute4Config:
+    metadata:
+      completion-handle:
+        get_resource: NovaCompute4CompletionHandle
+      hosts:
+        list_join:
+        - '
+
+          '
+        - - list_join:
+            - ' '
+            - - Fn::Select:
+                - 0
+                - Fn::Select:
+                  - ctlplane
+                  - get_attr:
+                    - NovaCompute0
+                    - networks
+              - Fn::Select:
+                - name
+                - get_attr:
+                  - NovaCompute0
+                  - show
+              - list_join:
+                - .
+                - - Fn::Select:
+                    - name
+                    - get_attr:
+                      - NovaCompute0
+                      - show
+                  - local
+          - list_join:
+            - ' '
+            - - Fn::Select:
+                - 0
+                - Fn::Select:
+                  - ctlplane
+                  - get_attr:
+                    - NovaCompute3
+                    - networks
+              - Fn::Select:
+                - name
+                - get_attr:
+                  - NovaCompute3
+                  - show
+              - list_join:
+                - .
+                - - Fn::Select:
+                    - name
+                    - get_attr:
+                      - NovaCompute3
+                      - show
+                  - local
+          - list_join:
+            - ' '
+            - - Fn::Select:
+                - 0
+                - Fn::Select:
+                  - ctlplane
+                  - get_attr:
+                    - NovaCompute4
+                    - networks
+              - Fn::Select:
+                - name
+                - get_attr:
+                  - NovaCompute4
+                  - show
+              - list_join:
+                - .
+                - - Fn::Select:
+                    - name
+                    - get_attr:
+                      - NovaCompute4
+                      - show
+                  - local
+      os-collect-config:
+        cfn:
+          access_key_id:
+            get_resource: NovaCompute4Key
+          path: NovaCompute4Config.Metadata
+          secret_access_key:
+            get_attr:
+            - NovaCompute4Key
+            - SecretAccessKey
+          stack_name:
+            get_param: AWS::StackName
+    type: AWS::AutoScaling::LaunchConfiguration
+  NovaCompute4Key:
+    properties:
+      UserName:
+        get_param: ComputeUser
+    type: AWS::IAM::AccessKey

test_merge.bash

@@ -36,6 +36,8 @@ run_test "python $merge_py --hot examples/source_include_subkey_hot.yaml" exampl
 run_test "python $merge_py --hot examples/launchconfig1_hot.yaml examples/launchconfig2_hot.yaml" examples/launchconfig_result_hot.yaml
 run_test "python $merge_py --hot --scale NovaCompute=3 examples/scale1_hot.yaml" examples/scale_result_hot.yaml
 run_test "python $merge_py --hot --scale NovaCompute=3 examples/scale_map_hot.yaml" examples/scale_map_result_hot.yaml
+run_test "python $merge_py --hot --scale NovaCompute=5,1,2 examples/scale_map_hot.yaml" examples/scale_map_result_hot_blacklist.yaml
+run_test "python $merge_py --hot --scale NovaCompute=3, examples/scale_map_hot.yaml" examples/scale_map_result_hot.yaml
 echo
 trap - EXIT
 exit $fail

tripleo_heat_merge/merge.py

@@ -80,6 +80,8 @@ def apply_scaling(template, scaling, in_copies=None):
 
     in_copies is reset to None when a dict {'Merge::Map': someobject} is
     encountered.
+
+    :param scaling: A dict of prefix -> (count, blacklists).
     """
     in_copies = dict(in_copies or {})
     # Shouldn't be needed but to avoid unexpected side effects/bugs we short
@@ -125,7 +127,7 @@ def scale_value(value, scaling, in_copies):
     """Scale out a value.
 
     :param value: The value to scale (not a container).
-    :param scaling: The scaling map to use.
+    :param scaling: The scaling map (prefix-> (copies, blacklist) to use.
     :param in_copies: What containers we're currently copying.
     :return: An iterator of the new values for the value as tuples:
         (prefix, copy_num, value). E.g. Compute0, 1, Compute1Foo
@@ -134,7 +136,7 @@ def scale_value(value, scaling, in_copies):
          - and that prefix is not in in_copies
     """
     if isinstance(value, (str, unicode)):
-        for prefix, copies in scaling.items():
+        for prefix, (copies, blacklist) in scaling.items():
             if not value.startswith(prefix):
                 continue
             suffix = value[len(prefix):]
@@ -144,7 +146,8 @@ def scale_value(value, scaling, in_copies):
                 return
             else:
                 for n in range(copies):
-                    yield prefix, n, prefix[:-1] + str(n) + suffix
+                    if n not in blacklist:
+                        yield prefix, n, prefix[:-1] + str(n) + suffix
                 return
         yield None, None, value
     else:
@@ -156,9 +159,11 @@ def parse_scaling(scaling_args):
     scaling_args = scaling_args or []
     result = {}
     for item in scaling_args:
-        key, value = item.split('=')
-        value = int(value)
-        result[key + '0'] = value
+        key, values = item.split('=')
+        values = values.split(',')
+        value = int(values[0])
+        blacklist = frozenset(int(v) for v in values[1:] if v)
+        result[key + '0'] = value, blacklist
     return result
 
 
@@ -251,10 +256,12 @@ def main(argv=None):
                         help='File to write output to. - for stdout',
                         default='-')
     parser.add_argument('--scale', action="append",
-        help="Names to scale out. Pass Prefix=1 to cause a key Prefix0Foo to "
+        help="Names to scale out. Pass Prefix=2 to cause a key Prefix0Foo to "
         "be copied to Prefix1Foo in the output, and value Prefix0Bar to be"
         "renamed to Prefix1Bar inside that copy, or copied to Prefix1Bar "
-        "outside of any copy.")
+        "outside of any copy. Pass Prefix=3,1 to cause Prefix1* to be elided"
+        "when scaling Prefix out. Prefix=4,1,2 will likewise elide Prefix1 and"
+        "Prefix2.")
     parser.add_argument(
         '--change-image-params', action='store_true', default=False,
         help="Change parameters in templates to match resource names. This was "