From 3a432b63c9ab21b97810896696d212328d738bc4 Mon Sep 17 00:00:00 2001 From: Sergey Abramov Date: Thu, 9 Jun 2016 13:08:46 +0300 Subject: [PATCH] Add patch-active-img command Required for creating new patched bootstrap image based on active bootstrap image. It patches active bootstrap image to skip the step that add UEFI. Note: * You should active your patched image using command: fuel-bootstrap activate * squashfs-tools should be installed on master node yum install squashfs-tools Usage: octane patch-active-img Closes-bug: 1575054 Change-Id: I7b12db209b5d2db158d4c26bc162bab504aa1590 --- octane/commands/patch_active_image.py | 104 +++++++++++++++++++++ octane/magic_consts.py | 1 + octane/patches/fuel_agent/patch | 4 +- octane/tests/test_patch_active_img.py | 128 ++++++++++++++++++++++++++ setup.cfg | 1 + 5 files changed, 236 insertions(+), 2 deletions(-) create mode 100644 octane/commands/patch_active_image.py create mode 100644 octane/tests/test_patch_active_img.py diff --git a/octane/commands/patch_active_image.py b/octane/commands/patch_active_image.py new file mode 100644 index 00000000..4ee3c24b --- /dev/null +++ b/octane/commands/patch_active_image.py @@ -0,0 +1,104 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import hashlib +import logging +import os +import tarfile +import tempfile +import uuid +import yaml + +from cliff import command + +from octane import magic_consts +from octane.util import patch +from octane.util import subprocess +from octane.util import tempfile as temp_util + + +LOG = logging.getLogger(__name__) +IMAGE_LABEL = "patched_image" + + +def _patch_squashfs(root_img, patched_img, *patches): + with temp_util.temp_dir() as patch_dir: + LOG.info("unsquash root image to temporary directory") + subprocess.call(["unsquashfs", "-f", "-d", patch_dir, root_img]) + LOG.info("apply patch to root image") + patch.patch_apply(patch_dir, patches) + LOG.info("create new root.squashfs image") + subprocess.call(["mksquashfs", patch_dir, patched_img]) + + +def calculate_md5(filename): + md5 = hashlib.md5() + chunk_size = 4048 + with open(filename, "rb") as f: + block = f.read(chunk_size) + while block: + md5.update(block) + block = f.read(chunk_size) + return md5.hexdigest() + + +def _mk_metadata(src, dst, root_fs_path): + with open(src) as fd: + metadata = yaml.load(fd) + + uuid_val = metadata["uuid"] + my_uuid_val = str(uuid.uuid1()) + metadata["label"] = IMAGE_LABEL + metadata["uuid"] = my_uuid_val + + for module in metadata["modules"].values(): + module["uri"] = module["uri"].replace(uuid_val, my_uuid_val) + + metadata["modules"]["rootfs"]["raw_size"] = os.path.getsize(root_fs_path) + metadata["modules"]["rootfs"]["raw_md5"] = calculate_md5(root_fs_path) + with open(dst, "w") as fd: + yaml.dump(metadata, fd) + + +def patch_img(): + root_img = os.path.join(magic_consts.ACTIVE_IMG_PATH, "root.squashfs") + active_metadata_path = os.path.join( + magic_consts.ACTIVE_IMG_PATH, "metadata.yaml") + patch_file = os.path.join(magic_consts.CWD, "patches/fuel_agent/patch") + path_archname_pairs = [(os.path.join(magic_consts.ACTIVE_IMG_PATH, p), p) + for p in ["vmlinuz", "initrd.img"]] + + with temp_util.temp_dir() as temp_dir: + patched_img = os.path.join(temp_dir, "root.squashfs") + patched_metadata_path = os.path.join(temp_dir, "metadata.yaml") + + _patch_squashfs(root_img, patched_img, patch_file) + _mk_metadata(active_metadata_path, patched_metadata_path, patched_img) + + path_archname_pairs.append((patched_img, "root.squashfs")) + path_archname_pairs.append((patched_metadata_path, "metadata.yaml")) + + with tempfile.NamedTemporaryFile() as archive_file: + with tarfile.open(name=archive_file.name, mode="w:gz") as archive: + for path, archname in path_archname_pairs: + archive.add(path, archname) + + LOG.info("Import image using fuel-bootstrap") + subprocess.call(["fuel-bootstrap", "import", archive_file.name]) + LOG.info("Activate image using `fuel-bootstrap activate`") + + +class PatchImgCommand(command.Command): + """Create patched bootstrap image with label `{0}`""".format(IMAGE_LABEL) + + def take_action(self, parsed_args): + patch_img() diff --git a/octane/magic_consts.py b/octane/magic_consts.py index 14542da4..3bb8590d 100644 --- a/octane/magic_consts.py +++ b/octane/magic_consts.py @@ -77,3 +77,4 @@ CONFIGDRIVE_PART_SIZE = 10 KEYSTONE_CONF = "/etc/keystone/keystone.conf" KEYSTONE_PASTE = "/etc/keystone/keystone-paste.ini" +ACTIVE_IMG_PATH = "/var/www/nailgun/bootstraps/active_bootstrap/" diff --git a/octane/patches/fuel_agent/patch b/octane/patches/fuel_agent/patch index 8d048f66..d9e387e4 100644 --- a/octane/patches/fuel_agent/patch +++ b/octane/patches/fuel_agent/patch @@ -1,5 +1,5 @@ ---- usr/lib/python2.7/dist-packages/fuel_agent/drivers/nailgun.py -+++ usr/lib/python2.7/dist-packages/fuel_agent/drivers/nailgun.py +--- a/usr/lib/python2.7/dist-packages/fuel_agent/drivers/nailgun.py ++++ b/usr/lib/python2.7/dist-packages/fuel_agent/drivers/nailgun.py @@ -321,10 +321,6 @@ LOG.debug('Adding bios_grub partition on disk %s: size=24' % disk['name']) diff --git a/octane/tests/test_patch_active_img.py b/octane/tests/test_patch_active_img.py new file mode 100644 index 00000000..561b93a2 --- /dev/null +++ b/octane/tests/test_patch_active_img.py @@ -0,0 +1,128 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import os +import pytest + +from octane.commands import patch_active_image +from octane import magic_consts + + +@pytest.mark.parametrize("root_img", ["root_img_path"]) +@pytest.mark.parametrize("patched_img", ["patch_img_path"]) +@pytest.mark.parametrize("patches", [("patch_1", ), ("patch_1", "patch_2")]) +def test_patch_squashfs(mocker, root_img, patched_img, patches): + temp_dir_mock = mocker.patch("octane.util.tempfile.temp_dir") + temp_dir_mock.return_value.__enter__ = temp_dir_mock + subprocess_mock = mocker.patch("octane.util.subprocess.call") + patch_apply_mock = mocker.patch("octane.util.patch.patch_apply") + patch_active_image._patch_squashfs(root_img, patched_img, *patches) + patch_apply_mock.assert_called_once_with( + temp_dir_mock.return_value, patches) + assert [ + mock.call([ + "unsquashfs", "-f", "-d", temp_dir_mock.return_value, root_img + ]), + mock.call([ + "mksquashfs", temp_dir_mock.return_value, patched_img + ]), + ] == subprocess_mock.call_args_list + + +@pytest.mark.parametrize("src", ["src_data"]) +@pytest.mark.parametrize("dst", ["dst_data"]) +@pytest.mark.parametrize("root_fs_path", ["root_fs_path"]) +def test_mk_metadata(mocker, mock_open, src, dst, root_fs_path): + os_size_mock = mocker.patch("os.path.getsize") + calc_md5 = mocker.patch("octane.commands.patch_active_image.calculate_md5") + test_uuid = "123-123" + uri = "path/{uuid}/qwer" + data = { + "uuid": test_uuid, + "label": "test_label", + "modules": { + "rootfs": { + "raw_size": 123123, + "raw_md5": 123123, + "uri": uri.format(uuid=test_uuid) + } + } + } + load_mock = mocker.patch("yaml.load", return_value=data) + dump_mock = mocker.patch("yaml.dump") + uuid_mock = mocker.patch("uuid.uuid1", return_value="my_generated_uuid") + results = data.copy() + patch_active_image._mk_metadata(src, dst, root_fs_path) + results["label"] = patch_active_image.IMAGE_LABEL + results["uuid"] = uuid_mock.return_value + results["modules"]["rootfs"]["raw_size"] = os_size_mock.return_value + results["modules"]["rootfs"]["raw_md5"] = calc_md5.return_value + results["modules"]["rootfs"]["uri"] = uri.format( + uuid=uuid_mock.return_value) + dump_mock.assert_called_once_with(results, mock_open.return_value) + assert [mock.call(src), mock.call(dst, "w")] == mock_open.call_args_list + load_mock.assert_called_once_with(mock_open.return_value) + + +@pytest.mark.parametrize("work_dir", ["/working_dir"]) +def test_patch_img(mocker, work_dir): + temp_dir_mock = mocker.patch("octane.util.tempfile.temp_dir") + temp_dir_mock.return_value.__enter__.return_value = work_dir + mock_patch_sqfs = mocker.patch( + "octane.commands.patch_active_image._patch_squashfs") + mock_patch_mk_metadata = mocker.patch( + "octane.commands.patch_active_image._mk_metadata") + mock_named_temp = mocker.patch("tempfile.NamedTemporaryFile") + mock_named_temp.return_value.__enter__.return_value = \ + mock_named_temp.return_value + mock_tarfile = mocker.patch("tarfile.open") + mock_tarfile.return_value.__enter__.return_value = \ + mock_tarfile.return_value + subprocess_mock = mocker.patch("octane.util.subprocess.call") + + root_img = os.path.join(magic_consts.ACTIVE_IMG_PATH, "root.squashfs") + patch_file = os.path.join(magic_consts.CWD, "patches/fuel_agent/patch") + patched_img = os.path.join(work_dir, "root.squashfs") + + patch_active_image.patch_img() + + mock_patch_sqfs.assert_called_once_with(root_img, patched_img, patch_file) + mock_patch_mk_metadata.assert_called_once_with( + os.path.join(magic_consts.ACTIVE_IMG_PATH, "metadata.yaml"), + os.path.join(work_dir, "metadata.yaml"), + patched_img + ) + mock_named_temp.assert_called_once_with() + mock_tarfile.assert_called_once_with( + name=mock_named_temp.return_value.name, + mode="w:gz") + arch_add_calls = ([ + mock.call(os.path.join(work_dir, "metadata.yaml"), "metadata.yaml"), + mock.call(os.path.join(work_dir, "root.squashfs"), "root.squashfs"), + ] + [ + mock.call(os.path.join(magic_consts.ACTIVE_IMG_PATH, p), p) + for p in ["vmlinuz", "initrd.img"] + ]) + mock_tarfile.return_value.add.assert_has_calls( + arch_add_calls, any_order=True) + + subprocess_mock.assert_called_once_with([ + "fuel-bootstrap", "import", mock_named_temp.return_value.name + ]) + + +def test_parser(mocker, octane_app): + patch_img_mock = mocker.patch( + "octane.commands.patch_active_image.patch_img") + octane_app.run(["patch-active-img"]) + patch_img_mock.assert_called_once_with() diff --git a/setup.cfg b/setup.cfg index dc84f7c5..9d344b57 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,6 +41,7 @@ octane = fuel-repo-restore = octane.commands.restore:RestoreRepoCommand update-bootstrap-centos = octane.commands.update_bootstrap:UpdateCentos enable-release = octane.commands.enable_release:EnableReleaseCommand + patch-active-img = octane.commands.patch_active_image:PatchImgCommand octane.handlers.upgrade = controller = octane.handlers.upgrade.controller:ControllerUpgrade compute = octane.handlers.upgrade.compute:ComputeUpgrade