Support including and excluding files from config save

Currently in the upgrade job we are seeing the OOM killer kick in during
the 'overcloud service configuration save' command. Ansible is quite
inefficient when copying large files around, so excluding the large IPA
images should relieve some memory pressure.

Change-Id: I3a230b0a699154606ca8faa00a85d45ae815c599
Story: 2004704
Task: 28733

ansible/overcloud-service-config-save.yml

@@ -15,6 +15,8 @@
       find:
         paths: "{{ node_config_directory }}"
         recurse: True
+        excludes: "{{ exclude_patterns | default(omit) }}"
+        patterns: "{{ include_patterns | default(omit) }}"
       register: find_result
       become: true
 

dev/functions

@@ -353,11 +353,9 @@ function overcloud_upgrade {
     fi
 
     echo "Saving overcloud service configuration"
-    if ! run_kayobe overcloud service configuration save; then
+    # Don't copy the ironic IPA kernel and ramdisk, since these files can be
-        # NOTE(mgoddard): This fails in CI due to a memory error while copying
+    # quite large.
-        # the IPA deployment images.
+    run_kayobe overcloud service configuration save --exclude 'ironic-agent.*'
-        echo "FIXME: Saving service configuration failed. Ignoring for now"
-    fi
 
     echo "Deploying containerised overcloud services"
     run_kayobe overcloud service upgrade

kayobe/cli/commands.py

@@ -952,6 +952,12 @@ class OvercloudServiceConfigurationSave(KayobeAnsibleMixin, VaultMixin,
         parser = super(OvercloudServiceConfigurationSave, self).get_parser(
             prog_name)
         group = parser.add_argument_group("Service configuration")
+        group.add_argument("--exclude",
+                           help="optional comma-separated list of patterns "
+                                "matching filenames to exclude")
+        group.add_argument("--include",
+                           help="optional comma-separated list of patterns "
+                                "matching filenames to include")
         group.add_argument("--node-config-dir",
                            help="the directory to store the config files on "
                                 "the remote node (default /etc/kolla)")
@@ -964,6 +970,10 @@ class OvercloudServiceConfigurationSave(KayobeAnsibleMixin, VaultMixin,
         self.app.LOG.debug("Saving overcloud service configuration")
         playbooks = _build_playbook_list("overcloud-service-config-save")
         extra_vars = {}
+        if parsed_args.exclude:
+            extra_vars["exclude_patterns"] = parsed_args.exclude
+        if parsed_args.include:
+            extra_vars["include_patterns"] = parsed_args.include
         if parsed_args.output_dir:
             extra_vars["config_save_path"] = parsed_args.output_dir
         if parsed_args.node_config_dir:

kayobe/tests/unit/cli/test_commands.py

@@ -1165,6 +1165,56 @@ class TestCase(unittest.TestCase):
         ]
         self.assertEqual(expected_calls, mock_run.call_args_list)
 
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    def test_overcloud_service_configuration_save(self, mock_run):
+        command = commands.OvercloudServiceConfigurationSave(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path(
+                        "ansible", "overcloud-service-config-save.yml"),
+                ],
+                extra_vars={}
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    def test_overcloud_service_configuration_save_args(self, mock_run):
+        command = commands.OvercloudServiceConfigurationSave(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([
+            "--exclude", "exclude1,exclude2",
+            "--include", "include1,include2",
+            "--node-config-dir", "/path/to/config",
+            "--output-dir", "/path/to/output",
+        ])
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path(
+                        "ansible", "overcloud-service-config-save.yml"),
+                ],
+                extra_vars={
+                    "exclude_patterns": "exclude1,exclude2",
+                    "include_patterns": "include1,include2",
+                    "config_save_path": "/path/to/output",
+                    "node_config_directory": "/path/to/config",
+                }
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
     def test_overcloud_container_image_build(self, mock_run):

releasenotes/notes/save-include-exclude-df8cca17da0aa3de.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds support for including or excluding files from the output of ``kayobe
+    overcloud service configuration save``. This is particularly useful for
+    large files such as the Ironic IPA images.