From 1a8801a8702555caf9e45fde902ab6037e2c59e7 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Fri, 30 Oct 2015 22:06:42 +0300 Subject: [PATCH] Deprecate version.yaml At the moment feature_groups are defined during ISO build in version.yaml file. We need to avoid using build time defined parameters because we are moving to the package based delivery approach. Depends-On: I05e841d4279bc61645cefb68f411a93b67e57189 Related-Bug: #1513737 Related-Bug: #1515517 Change-Id: I4f96fec73f991094bca65de5f1d1014662fbb2b4 --- nailgun/nailgun/api/v1/handlers/version.py | 6 ---- nailgun/nailgun/consts.py | 13 ++++++++ nailgun/nailgun/settings.py | 33 +++++++++++++------ nailgun/nailgun/settings.yaml | 18 ++++------ .../fuel_statistics/installation_info.py | 24 +++++++++++--- .../test_master_node_settings_handler.py | 12 +++++++ .../test/integration/test_public_api.py | 28 ---------------- .../test_installation_info.py | 12 +++++++ nailgun/nailgun/test/unit/test_utils.py | 22 ------------- .../nailgun/test/unit/test_version_handler.py | 15 ++------- nailgun/nailgun/utils/__init__.py | 25 -------------- specs/fuel-nailgun.spec | 2 ++ 12 files changed, 90 insertions(+), 120 deletions(-) diff --git a/nailgun/nailgun/api/v1/handlers/version.py b/nailgun/nailgun/api/v1/handlers/version.py index ed99ba8f71..975678abc0 100644 --- a/nailgun/nailgun/api/v1/handlers/version.py +++ b/nailgun/nailgun/api/v1/handlers/version.py @@ -21,14 +21,11 @@ Product info handlers from nailgun.api.v1.handlers.base import BaseHandler from nailgun.api.v1.handlers.base import content from nailgun.settings import settings -from nailgun import utils class VersionHandler(BaseHandler): """Version info handler""" - release_versions = "/etc/fuel/release_versions/*.yaml" - @content def GET(self): """:returns: FUEL/FUELWeb commit SHA, release version. @@ -38,7 +35,4 @@ class VersionHandler(BaseHandler): version = settings.VERSION method = settings.AUTH['AUTHENTICATION_METHOD'] version['auth_required'] = method in ['fake', 'keystone'] - - version['release_versions'] = utils.get_fuel_release_versions( - self.release_versions) return version diff --git a/nailgun/nailgun/consts.py b/nailgun/nailgun/consts.py index 5d7b1c459f..69f625b4f6 100644 --- a/nailgun/nailgun/consts.py +++ b/nailgun/nailgun/consts.py @@ -382,6 +382,12 @@ FUEL_IMAGE_BASED_ONLY = '7.0' # version of Fuel when multiple floating IP ranges support is added FUEL_MULTIPLE_FLOATING_IP_RANGES = '8.0' +# this file is provided by the fuel-release package +FUEL_RELEASE_FILE = '/etc/fuel_release' + +# this file is provided by the fuel-openstack-metadata package +FUEL_OPENSTACK_VERSION_FILE = '/etc/fuel_openstack_version' + OSWL_RESOURCE_TYPES = Enum( 'vm', 'tenant', @@ -418,3 +424,10 @@ COMPONENT_TYPES = Enum( 'storage', 'additional_service', ) + +# NOTE(kozhukalov): This constant is used to collect +# the information about installed fuel packages (rpm -q). +# This information is necessary for fuel-stats. +STAT_FUEL_PACKAGES = ( + 'fuel-nailgun', +) diff --git a/nailgun/nailgun/settings.py b/nailgun/nailgun/settings.py index 81fc88be0b..eee08c94fc 100644 --- a/nailgun/nailgun/settings.py +++ b/nailgun/nailgun/settings.py @@ -20,6 +20,7 @@ import os import six import yaml +from nailgun import consts from nailgun.logger import logger @@ -33,16 +34,6 @@ class NailgunSettings(object): settings_files.append(project_settings_file) settings_files.append('/etc/nailgun/settings.yaml') - version_paths = ["/etc/fuel/version.yaml", - "/etc/fuel/nailgun/version.yaml", - "/etc/nailgun/version.yaml"] - for path in version_paths: - if os.access(path, os.R_OK): - settings_files.append(path) - break - else: - logger.error("'version.yaml' config file is not found") - test_config = os.environ.get('NAILGUN_CONFIG') if test_config: settings_files.append(test_config) @@ -56,6 +47,20 @@ class NailgunSettings(object): logger.error("Error while reading config file %s: %s" % (sf, str(e))) + self.config['VERSION']['api'] = self.config['API'] + self.config['VERSION']['feature_groups'] = \ + self.config['FEATURE_GROUPS'] + + fuel_release = self.get_file_content(consts.FUEL_RELEASE_FILE) + if fuel_release: + self.config['VERSION']['release'] = fuel_release + + fuel_openstack_version = self.get_file_content( + consts.FUEL_OPENSTACK_VERSION_FILE) + if fuel_openstack_version: + self.config['VERSION']['openstack_version'] = \ + fuel_openstack_version + if int(self.config.get("DEVELOPMENT")): logger.info("DEVELOPMENT MODE ON:") here = os.path.abspath( @@ -81,6 +86,14 @@ class NailgunSettings(object): yaml.load(custom_config.read()) ) + def get_file_content(self, path): + try: + with open(path, "r") as f: + return f.read().strip() + except Exception as e: + logger.error("Error while reading file: %s. %s", + path, six.text_type(e)) + def dump(self): return yaml.dump(self.config) diff --git a/nailgun/nailgun/settings.yaml b/nailgun/nailgun/settings.yaml index 2d7132f077..350b83457f 100644 --- a/nailgun/nailgun/settings.yaml +++ b/nailgun/nailgun/settings.yaml @@ -18,15 +18,14 @@ AUTH: VERSION: + # this value is still here for backward compatibility release: "8.0" - api: "1" - nailgun_sha: "Unknown build" - astute_sha: "Unknown build" - fuellib_sha: "Unknown build" - ostf_sha: "Unknown build" - feature_groups: - - experimental - - advanced + +API: "1" + +FEATURE_GROUPS: + - "experimental" + - "advanced" FUEL_KEY: "Unknown" @@ -150,9 +149,6 @@ OPENSTACK_INFO_COLLECTOR_PROXY_PORT: 8888 # (is used for OpenStack info collector) AUTH_PORT: 5000 -FUEL_VERSION_FILE: "/etc/fuel/version.yaml" -FUEL_VERSION_KEY: "VERSION" - PATH_TO_SSH_KEY: = "/root/.ssh/id_rsa" PATH_TO_BOOTSTRAP_SSH_KEY: "/root/.ssh/bootstrap.rsa" AUTHORIZED_KEYS: [] diff --git a/nailgun/nailgun/statistics/fuel_statistics/installation_info.py b/nailgun/nailgun/statistics/fuel_statistics/installation_info.py index 1aa980a540..dcc041f53d 100644 --- a/nailgun/nailgun/statistics/fuel_statistics/installation_info.py +++ b/nailgun/nailgun/statistics/fuel_statistics/installation_info.py @@ -13,9 +13,12 @@ # under the License. import copy +import subprocess +from nailgun import consts from nailgun.db.sqlalchemy.models import NeutronConfig from nailgun.db.sqlalchemy.models import NovaNetworkConfig +from nailgun.logger import logger from nailgun.objects import Cluster from nailgun.objects import ClusterCollection from nailgun.objects import MasterNodeSettings @@ -24,7 +27,6 @@ from nailgun.objects.plugin import ClusterPlugins from nailgun.settings import settings from nailgun.statistics.utils import get_attr_value from nailgun.statistics.utils import WhiteListRule -from nailgun import utils class InstallationInfo(object): @@ -129,10 +131,21 @@ class InstallationInfo(object): ) def fuel_release_info(self): - versions = utils.get_fuel_release_versions(settings.FUEL_VERSION_FILE) - if settings.FUEL_VERSION_KEY not in versions: - versions[settings.FUEL_VERSION_KEY] = settings.VERSION - return versions[settings.FUEL_VERSION_KEY] + return settings.VERSION + + def fuel_packages_info(self): + command = ['rpm', '-q'] + command.extend(consts.STAT_FUEL_PACKAGES) + p = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate() + if p.poll() != 0: + logger.error("Command '%s' failed. Error: %s", + " ".join(command), err) + return [] + return out.strip().split() def get_network_configuration_info(self, cluster): network_config = cluster.network_config @@ -299,6 +312,7 @@ class InstallationInfo(object): 'user_information': self.get_user_info(), 'master_node_uid': self.get_master_node_uid(), 'fuel_release': self.fuel_release_info(), + 'fuel_packages': self.fuel_packages_info(), 'clusters': clusters_info, 'clusters_num': len(clusters_info), 'allocated_nodes_num': allocated_nodes_num, diff --git a/nailgun/nailgun/test/integration/test_master_node_settings_handler.py b/nailgun/nailgun/test/integration/test_master_node_settings_handler.py index 5d627c87c5..a3a91847b4 100644 --- a/nailgun/nailgun/test/integration/test_master_node_settings_handler.py +++ b/nailgun/nailgun/test/integration/test_master_node_settings_handler.py @@ -13,6 +13,7 @@ # under the License. import copy +import mock from oslo_serialization import jsonutils @@ -25,6 +26,17 @@ from nailgun.utils import reverse class TestMasterNodeSettingsHandler(BaseMasterNodeSettignsTest): + def setUp(self): + self.patcher = mock.patch( + 'nailgun.statistics.fuel_statistics.installation_info' + '.InstallationInfo.fuel_packages_info', return_value=[]) + self.patcher.start() + super(TestMasterNodeSettingsHandler, self).setUp() + + def tearDown(self): + super(TestMasterNodeSettingsHandler, self).tearDown() + self.patcher.stop() + def test_get_controller(self): expected = self.master_node_settings diff --git a/nailgun/nailgun/test/integration/test_public_api.py b/nailgun/nailgun/test/integration/test_public_api.py index aec8f61aec..13e54919d3 100644 --- a/nailgun/nailgun/test/integration/test_public_api.py +++ b/nailgun/nailgun/test/integration/test_public_api.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -from mock import patch from oslo_serialization import jsonutils from nailgun.test.base import BaseAuthenticationIntegrationTest @@ -54,30 +53,3 @@ class TestPublicHandlers(BaseAuthenticationIntegrationTest): headers=self.default_headers ) self.assertEqual(200, resp.status_code) - - @patch('nailgun.api.v1.handlers.version.utils.get_fuel_release_versions') - def test_500_no_html_dev(self, handler_get): - exc_text = "Here goes an exception" - handler_get.side_effect = Exception(exc_text) - resp = self.app.get( - reverse('VersionHandler'), - headers=self.default_headers, - expect_errors=True - ) - self.assertEqual(500, resp.status_code) - self.assertIn(exc_text, resp.body) - self.assertIn("Traceback", resp.body) - self.assertNotIn("html", resp.body) - - @patch('nailgun.api.v1.handlers.version.utils.get_fuel_release_versions') - def test_500_no_html_production(self, handler_get): - exc_text = "Here goes an exception" - handler_get.side_effect = Exception(exc_text) - with patch('nailgun.settings.settings.DEVELOPMENT', 0): - resp = self.app.get( - reverse('VersionHandler'), - headers=self.default_headers, - expect_errors=True - ) - self.assertEqual(500, resp.status_code) - self.assertEqual(exc_text, resp.body) diff --git a/nailgun/nailgun/test/unit/fuel_statistics_tests/test_installation_info.py b/nailgun/nailgun/test/unit/fuel_statistics_tests/test_installation_info.py index 6fabf05900..eddfad325a 100644 --- a/nailgun/nailgun/test/unit/fuel_statistics_tests/test_installation_info.py +++ b/nailgun/nailgun/test/unit/fuel_statistics_tests/test_installation_info.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import mock import six from sqlalchemy.inspection import inspect @@ -30,6 +31,17 @@ from nailgun.statistics.fuel_statistics.installation_info \ class TestInstallationInfo(BaseTestCase): + def setUp(self): + self.patcher = mock.patch( + 'nailgun.statistics.fuel_statistics.installation_info' + '.InstallationInfo.fuel_packages_info', return_value=[]) + self.patcher.start() + super(TestInstallationInfo, self).setUp() + + def tearDown(self): + super(TestInstallationInfo, self).tearDown() + self.patcher.stop() + def test_release_info(self): info = InstallationInfo() f_info = info.fuel_release_info() diff --git a/nailgun/nailgun/test/unit/test_utils.py b/nailgun/nailgun/test/unit/test_utils.py index fe57a01afe..d3a47d322e 100644 --- a/nailgun/nailgun/test/unit/test_utils.py +++ b/nailgun/nailgun/test/unit/test_utils.py @@ -15,10 +15,7 @@ # under the License. import mock -from mock import mock_open from mock import patch -import os -import tempfile import requests @@ -27,7 +24,6 @@ from nailgun.utils import camel_to_snake_case from nailgun.utils import compact from nailgun.utils import dict_merge from nailgun.utils import flatten -from nailgun.utils import get_fuel_release_versions from nailgun.utils import grouper from nailgun.utils import http_get from nailgun.utils import traverse @@ -59,24 +55,6 @@ class TestUtils(base.BaseIntegrationTest): "dict": {"stuff": "hz", "another_stuff": "hz"}}}) - @patch('nailgun.utils.glob.glob', return_value=['test.yaml']) - @patch('__builtin__.open', mock_open(read_data='test_data')) - def test_get_release_versions(self, _): - versions = get_fuel_release_versions(None) - self.assertDictEqual({'test': 'test_data'}, versions) - - def test_get_release_versions_empty_file(self): - with tempfile.NamedTemporaryFile() as tf: - versions = get_fuel_release_versions(tf.name) - self.assertDictEqual({os.path.basename(tf.name): None}, versions) - - def test_get_release_no_file(self): - with tempfile.NamedTemporaryFile() as tf: - file_path = tf.name - self.assertFalse(os.path.exists(file_path)) - versions = get_fuel_release_versions(file_path) - self.assertDictEqual({}, versions) - def test_camel_case_to_snake_case(self): self.assertTrue( camel_to_snake_case('TestCase') == 'test_case') diff --git a/nailgun/nailgun/test/unit/test_version_handler.py b/nailgun/nailgun/test/unit/test_version_handler.py index 30e9fdf0da..6583250d5e 100644 --- a/nailgun/nailgun/test/unit/test_version_handler.py +++ b/nailgun/nailgun/test/unit/test_version_handler.py @@ -27,11 +27,7 @@ from nailgun.utils import reverse class TestVersionHandler(BaseIntegrationTest): @patch('nailgun.api.v1.handlers.version.settings.VERSION', { - 'release': '0.1b', - 'nailgun_sha': '12345', - "astute_sha": "Unknown build", - "fuellib_sha": "Unknown build", - "ostf_sha": "Unknown build", + "release": "0.1b", "feature_groups": [], }) def test_version_handler(self): @@ -55,14 +51,7 @@ class TestVersionHandler(BaseIntegrationTest): resp.json_body, { "release": "0.1b", - "nailgun_sha": "12345", - "astute_sha": "Unknown build", - "fuellib_sha": "Unknown build", - "ostf_sha": "Unknown build", - "auth_required": True, "feature_groups": [], - "release_versions": { - "test": "test_data" - } + "auth_required": True, } ) diff --git a/nailgun/nailgun/utils/__init__.py b/nailgun/nailgun/utils/__init__.py index 41c8c141f5..cebb891736 100644 --- a/nailgun/nailgun/utils/__init__.py +++ b/nailgun/nailgun/utils/__init__.py @@ -193,31 +193,6 @@ class AttributesGenerator(object): ) return base64.b64encode(header + key) -def get_fuel_release_versions(path_mask): - """Returns release versions from files match to path_mask - - :param path_mask: mask of path to release version files - :type path_mask: string - :returns: dicts with file names as keys and release - versions info as values - """ - - result = {} - for fl in glob.glob(path_mask): - with open(fl, "r") as release_version: - file_name = os.path.splitext(os.path.basename(fl))[0] - try: - result[file_name] = yaml.load(release_version.read()) - except Exception as exc: - logger.warning( - u"Failed to load release version " - "info from '{0}': {1}".format( - fl, - six.text_type(exc) - ) - ) - return result - def camel_to_snake_case(name): """Convert camel case format into snake case diff --git a/specs/fuel-nailgun.spec b/specs/fuel-nailgun.spec index 46b1888fda..b8b5eb24d2 100644 --- a/specs/fuel-nailgun.spec +++ b/specs/fuel-nailgun.spec @@ -17,6 +17,8 @@ BuildRequires: python-yaml BuildRequires: git #BuildRequires: nodejs BuildArch: noarch +Requires: fuel-openstack-metadata +Requires: fuel-release Requires: python-alembic >= 0.6.2 Requires: python-amqplib >= 1.0.2 Requires: python-anyjson >= 0.3.3