From 857cc39d6358e02fed78bfd0d1a5a785469e1881 Mon Sep 17 00:00:00 2001 From: Evgeniy L Date: Thu, 18 Dec 2014 19:56:17 +0400 Subject: [PATCH] fpb, generate checksums of each file in the plugin Without checksums file it's impossible to find out if plugin was changed after installation. Change-Id: Idf66bf1e5c3a39ef55ba61295b55c2bbf0e40748 Closes-bug: #1403960 --- fuel_plugin_builder/CHANGELOG.md | 2 + .../fuel_plugin_builder/actions/build.py | 5 ++ .../fuel_plugin_builder/tests/base.py | 3 ++ .../fuel_plugin_builder/tests/test_build.py | 9 ++++ .../fuel_plugin_builder/tests/test_utils.py | 49 +++++++++++++++++ .../fuel_plugin_builder/utils.py | 53 +++++++++++++++++++ 6 files changed, 121 insertions(+) diff --git a/fuel_plugin_builder/CHANGELOG.md b/fuel_plugin_builder/CHANGELOG.md index 96d4c35..307c6e8 100644 --- a/fuel_plugin_builder/CHANGELOG.md +++ b/fuel_plugin_builder/CHANGELOG.md @@ -11,6 +11,8 @@ https://bugs.launchpad.net/fuel/+bug/1396491 - Improved validation for environment_config.yaml file, added required fields for attributes +- Generate file with SHA1 checksums for each file in the plugin + https://bugs.launchpad.net/fuel/+bug/1403960 ## 1.0.1 (2014-11-20) diff --git a/fuel_plugin_builder/fuel_plugin_builder/actions/build.py b/fuel_plugin_builder/fuel_plugin_builder/actions/build.py index 3b47ab2..f3febd2 100644 --- a/fuel_plugin_builder/fuel_plugin_builder/actions/build.py +++ b/fuel_plugin_builder/fuel_plugin_builder/actions/build.py @@ -35,18 +35,23 @@ class BuildPlugin(BaseAction): self.pre_build_hook_path = join_path(plugin_path, 'pre_build_hook') self.meta = utils.parse_yaml(join_path(plugin_path, 'metadata.yaml')) self.build_dir = join_path(plugin_path, '.build') + self.checksums_path = join_path(self.build_dir, 'checksums.sha1') def run(self): logger.debug('Start plugin building "%s"', self.plugin_path) self.run_pre_build_hook() self.check() self.build_repos() + self.add_checksums_file() self.make_package() def run_pre_build_hook(self): if utils.which(self.pre_build_hook_path): utils.exec_cmd(self.pre_build_hook_path) + def add_checksums_file(self): + utils.create_checksums_file(self.build_dir, self.checksums_path) + def make_package(self): full_name = '{0}-{1}'.format(self.meta['name'], self.meta['version']) diff --git a/fuel_plugin_builder/fuel_plugin_builder/tests/base.py b/fuel_plugin_builder/fuel_plugin_builder/tests/base.py index 71c417e..33d9d96 100644 --- a/fuel_plugin_builder/fuel_plugin_builder/tests/base.py +++ b/fuel_plugin_builder/fuel_plugin_builder/tests/base.py @@ -39,6 +39,9 @@ class FakeFile(StringIO): def __exit__(self, *args): pass + def writelines(self, lines): + self.write(''.join(lines)) + class BaseTestCase(TestCase): """Base class for test cases diff --git a/fuel_plugin_builder/fuel_plugin_builder/tests/test_build.py b/fuel_plugin_builder/fuel_plugin_builder/tests/test_build.py index afafe94..b93646e 100644 --- a/fuel_plugin_builder/fuel_plugin_builder/tests/test_build.py +++ b/fuel_plugin_builder/fuel_plugin_builder/tests/test_build.py @@ -47,6 +47,7 @@ class TestBuild(BaseTestCase): 'run_pre_build_hook', 'check', 'build_repos', + 'add_checksums_file', 'make_package'] self.mock_methods(self.builder, mocked_methods) @@ -54,6 +55,7 @@ class TestBuild(BaseTestCase): self.builder.run_pre_build_hook.assert_called_once_with() self.builder.check.assert_called_once_with() + self.builder.add_checksums_file() self.builder.build_repos.assert_called_once_with() self.builder.make_package() @@ -148,3 +150,10 @@ class TestBuild(BaseTestCase): manager_class_mock.assert_called_once_with(self.plugin_path) validator_manager_obj.get_validator.assert_called_once_with() validator_mock.validate.assert_called_once_with() + + @mock.patch( + 'fuel_plugin_builder.actions.build.utils.create_checksums_file') + def test_add_checksums_file(self, create_checksums_file_mock): + self.builder.add_checksums_file() + create_checksums_file_mock.assert_called_once_with( + self.builder.build_dir, self.builder.checksums_path) diff --git a/fuel_plugin_builder/fuel_plugin_builder/tests/test_utils.py b/fuel_plugin_builder/fuel_plugin_builder/tests/test_utils.py index 554d5cd..3e342a7 100644 --- a/fuel_plugin_builder/fuel_plugin_builder/tests/test_utils.py +++ b/fuel_plugin_builder/fuel_plugin_builder/tests/test_utils.py @@ -22,6 +22,7 @@ from mock import patch from fuel_plugin_builder import errors from fuel_plugin_builder.tests.base import BaseTestCase +from fuel_plugin_builder.tests.base import FakeFile from fuel_plugin_builder import utils @@ -250,3 +251,51 @@ class TestUtils(BaseTestCase): mock.call('/tmp/some_plugin/file4.mako', '/tmp/some_plugin/file4')], copy_permissions_mock.call_args_list) + + def test_calculate_sha(self): + file_path = '/tmp/file' + + with mock.patch('__builtin__.open', + self.mock_open('fake file content')): + + self.assertEqual( + utils.calculate_sha(file_path), + '5083c27641e7e4ae287d690cb3fafb4dd6e8f6ab') + + @mock.patch('fuel_plugin_builder.utils.calculate_sha') + @mock.patch('fuel_plugin_builder.utils.os.walk') + def test_calculate_checksums(self, walk_mock, sha_mock): + dir_path = '/tmp/dir_path' + walk_mock.return_value = [ + [dir_path, '', ['file1.txt', 'file2.txt']], + [dir_path, '', ['file3.txt']]] + + sha_mock.side_effect = ['sha_1', 'sha_2', 'sha_3'] + + self.assertEqual( + utils.calculate_checksums(dir_path), + [{'file_path': 'file1.txt', 'checksum': 'sha_1'}, + {'file_path': 'file2.txt', 'checksum': 'sha_2'}, + {'file_path': 'file3.txt', 'checksum': 'sha_3'}]) + + self.assertEqual( + [mock.call('/tmp/dir_path/file1.txt'), + mock.call('/tmp/dir_path/file2.txt'), + mock.call('/tmp/dir_path/file3.txt')], + sha_mock.call_args_list) + + @mock.patch('fuel_plugin_builder.utils.calculate_checksums') + def test_create_checksums_file(self, calculate_mock): + calculate_mock.return_value = [ + {'checksum': 'checksum2', 'file_path': 'file2.txt'}, + {'checksum': 'checksum', 'file_path': 'file1.txt'}] + + fileobj = FakeFile('') + open_mock = mock.MagicMock(return_value=fileobj) + + with mock.patch('__builtin__.open', open_mock): + utils.create_checksums_file('/tmp/dir', '/tmp/checksums') + + self.assertEqual( + fileobj.getvalue(), + 'checksum file1.txt\nchecksum2 file2.txt\n') diff --git a/fuel_plugin_builder/fuel_plugin_builder/utils.py b/fuel_plugin_builder/fuel_plugin_builder/utils.py index e40cedd..c102ba7 100644 --- a/fuel_plugin_builder/fuel_plugin_builder/utils.py +++ b/fuel_plugin_builder/fuel_plugin_builder/utils.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import hashlib import logging import os import shutil @@ -240,3 +241,55 @@ def parse_yaml(path): :returns: dict or list """ return yaml.load(open(path)) + + +def calculate_sha(file_path, chunk_size=2 ** 20): + """Calculate file's checksum + + :param str file_path: file path + :param int chunk_size: optional parameter, size of chunk + :returns: SHA1 string + """ + sha = hashlib.sha1() + + with open(file_path, 'rb') as f: + for chunk in iter(lambda: f.read(chunk_size), b''): + sha.update(chunk) + + return sha.hexdigest() + + +def calculate_checksums(dir_path): + """Calculates checksums of files in the directory + + :param str dir_path: path to the directory + :returns: list of dicts, where 'checksum' is SHA1, + 'file_path' is a relative path to the file + """ + checksums = [] + for root, _, files in os.walk(dir_path): + for file_path in files: + full_path = os.path.join(root, file_path) + rel_path = os.path.relpath(full_path, dir_path) + + checksums.append({ + 'checksum': calculate_sha(full_path), + 'file_path': rel_path}) + + return checksums + + +def create_checksums_file(dir_path, checksums_file): + """Creates file with checksums + + :param str dir_path: path to the directory for checksums calculation + :param str checksums_file: path to the file where checksums are saved + """ + checksums = calculate_checksums(dir_path) + checksums_sorted = sorted(checksums, key=lambda c: c['file_path']) + checksum_lines = [ + '{checksum} {file_path}\n'.format(**checksum) + for checksum in checksums_sorted] + + with open(checksums_file, 'w') as f: + f.writelines(checksum_lines)