Extension performance improvement
* Introduced Git repo stat TTL. This helps to avoid unnecessary fetch requests to remote repository * Configuration file restructured. It's easier to have mapping for file name in Git repository on resource rather then vise versa Change-Id: I222199fc8aa203966a465508d4ecd8477f0fac3e
This commit is contained in:
parent
40e1ede7b9
commit
0e58b576b3
|
@ -11,3 +11,4 @@
|
|||
# under the License.
|
||||
|
||||
REPOS_DIR = '/var/lib/fuel_repos'
|
||||
REPO_TTL = 1200
|
||||
|
|
|
@ -33,22 +33,12 @@ class OpenStackConfigPipeline(BasePipeline):
|
|||
|
||||
Genereate OpenStack configuration hash based on configuration files
|
||||
stored in git repository associated with a particular environment
|
||||
Example of configuration extension:
|
||||
configuration:
|
||||
nova_config:
|
||||
DEFAULT/debug:
|
||||
value: True
|
||||
DEFAULT/amqp_durable_queues:
|
||||
value: False
|
||||
keystone_config:
|
||||
DEFAULT/default_publisher_id:
|
||||
ensure: absent
|
||||
DEFAULT/crypt_strength:
|
||||
value: 6000
|
||||
"""
|
||||
logger.info("Started serialisation for node {}".format(node.id))
|
||||
repo = GitRepo.get_by_cluster_id(node.cluster_id)
|
||||
if not repo:
|
||||
return node_data
|
||||
|
||||
GitRepo.checkout(repo)
|
||||
repo_path = os.path.join(const.REPOS_DIR, repo.repo_name)
|
||||
resource_mapping = ExternalGit.ext_settings['resource_mapping']
|
||||
|
@ -99,14 +89,17 @@ class OpenStackConfigPipeline(BasePipeline):
|
|||
override_configs['nodes'].get(uid, {}))
|
||||
|
||||
node_data['configuration'] = common
|
||||
logger.info("Node {0} config from git {1}".format(uid, common))
|
||||
logger.debug("Node {0} config from git {1}".format(uid, common))
|
||||
logger.info("Finished serialisation for node {}".format(node.id))
|
||||
return node_data
|
||||
|
||||
@classmethod
|
||||
def process_deployment_for_cluster(self, cluster, data):
|
||||
logger.info("Started serialisation for cluster {}".format(cluster.id))
|
||||
repo = GitRepo.get_by_cluster_id(cluster.id)
|
||||
if not repo:
|
||||
return data
|
||||
|
||||
if repo.manage_master:
|
||||
GitRepo.checkout(repo)
|
||||
repo_path = os.path.join(const.REPOS_DIR, repo.repo_name)
|
||||
|
@ -118,11 +111,11 @@ class OpenStackConfigPipeline(BasePipeline):
|
|||
)
|
||||
|
||||
data['master_config'] = master_config
|
||||
|
||||
logger.info("Finished serialisation for cluster {}".format(cluster.id))
|
||||
return data
|
||||
|
||||
|
||||
# TODO(dukov) Remove decorator extension management is available
|
||||
@utils.register_extension(u'fuel_external_git')
|
||||
class ExternalGit(BaseExtension):
|
||||
name = 'fuel_external_git'
|
||||
version = '1.0.0'
|
||||
|
|
|
@ -12,8 +12,8 @@
|
|||
|
||||
single_schema = {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"title": "Cluster",
|
||||
"description": "Serialized Cluster object",
|
||||
"title": "GitRepo",
|
||||
"description": "Serialized GitRepo object",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "number"},
|
||||
|
|
|
@ -12,6 +12,7 @@
|
|||
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import yaml
|
||||
|
||||
from distutils.dir_util import copy_tree
|
||||
|
@ -22,6 +23,7 @@ from fuel_external_git.models import GitRepo
|
|||
from git import exc
|
||||
from git import Repo
|
||||
|
||||
from nailgun.consts import CLUSTER_STATUSES
|
||||
from nailgun.db import db
|
||||
from nailgun import errors
|
||||
from nailgun.logger import logger
|
||||
|
@ -92,6 +94,20 @@ class GitRepo(NailgunObject):
|
|||
|
||||
@classmethod
|
||||
def checkout(self, instance):
|
||||
fetch_file = os.path.join(
|
||||
const.REPOS_DIR,
|
||||
instance.repo_name,
|
||||
'.git/FETCH_HEAD'
|
||||
)
|
||||
if os.path.exists(fetch_file):
|
||||
current_ts = time.time()
|
||||
cluster = Cluster.get_by_uid(instance.env_id)
|
||||
last_fetch = os.stat(fetch_file).st_mtime
|
||||
if cluster.status != CLUSTER_STATUSES.deployment and \
|
||||
current_ts - last_fetch < const.REPO_TTL:
|
||||
return
|
||||
|
||||
logger.debug("Repo TTL exceeded. Fetching code...")
|
||||
ssh_cmd = self._get_ssh_cmd(instance.repo_name)
|
||||
|
||||
if not os.path.exists(self._get_key_path(instance.repo_name)):
|
||||
|
|
|
@ -1,69 +1,69 @@
|
|||
resource_mapping:
|
||||
ceilometer_api_paste_ini:
|
||||
alias: ceilometer-api-paste.ini
|
||||
path: /etc/ceilometer/api-paste.ini
|
||||
ceilometer:
|
||||
alias: ceilometer.conf
|
||||
path: /etc/ceilometer/ceilometer.conf
|
||||
cinder_api_paste_ini:
|
||||
alias: cinder-api-paste.ini
|
||||
path: /etc/cinder/api-paste.ini
|
||||
cinder:
|
||||
alias: cinder.conf
|
||||
path: /etc/cinder/cinder.conf
|
||||
glance_api:
|
||||
alias: glance-api.conf
|
||||
path: /etc/glance/glance-api.conf
|
||||
glance_cache:
|
||||
alias: glance-cache.conf
|
||||
path: /etc/glance/glance-cache.conf
|
||||
glare_config:
|
||||
alias: glance-glare.conf
|
||||
path: /etc/glance/glance-glare.conf
|
||||
glance_registry:
|
||||
alias: glance-registry.conf
|
||||
path: /etc/glance/glance-registry.conf
|
||||
heat_api_paste_ini:
|
||||
alias: heat-api-paste.ini
|
||||
path: /etc/heat/api-paste.ini
|
||||
heat:
|
||||
alias: heat.conf
|
||||
path: /etc/heat/heat.conf
|
||||
keystone_config:
|
||||
alias: keystone.conf
|
||||
path: /etc/keystone/keystone.conf
|
||||
neutron_agent_ovs:
|
||||
alias: openvswitch_agent.ini
|
||||
path: /etc/neutron/plugins/ml2/openvswitch_agent.ini
|
||||
neutron_api_config:
|
||||
alias: neutron-api-paste.ini
|
||||
path: /etc/neutron/api-paste.ini
|
||||
neutron_config:
|
||||
alias: neutron.conf
|
||||
path: /etc/neutron/neutron.conf
|
||||
neutron_dhcp_agent_config:
|
||||
alias: dhcp_agent.ini
|
||||
path: /etc/neutron/dhcp_agent.ini
|
||||
neutron_l3_agent_config:
|
||||
alias: l3_agent.ini
|
||||
path: /etc/neutron/l3_agent.ini
|
||||
neutron_metadata_agent_config:
|
||||
alias: metadata_agent.ini
|
||||
path: /etc/neutron/metadata_agent.ini
|
||||
neutron_plugin_ml2:
|
||||
alias: ml2_conf.ini
|
||||
path: /etc/neutron/plugins/ml2/ml2_conf.ini
|
||||
neutron_sriov_agent_config:
|
||||
alias: sriov_agent.ini
|
||||
path: /etc/neutron/plugins/ml2/sriov_agent.ini
|
||||
nova_config:
|
||||
alias: nova.conf
|
||||
path: /etc/nova/nova.conf
|
||||
nova_paste_api_ini:
|
||||
alias: nova-api-paste.ini
|
||||
path: /etc/nova/api-paste.ini
|
||||
master_mapping:
|
||||
master_config:
|
||||
alias: master_config.yaml
|
||||
path: ""
|
||||
driver: 'fuel_external_git.drivers.yaml_driver.YamlConfig'
|
||||
master_config.yaml:
|
||||
resource: master_config
|
||||
driver: fuel_external_git.drivers.yaml_driver.YamlConfig
|
||||
path: ''
|
||||
resource_mapping:
|
||||
ceilometer-api-paste.ini:
|
||||
path: /etc/ceilometer/api-paste.ini
|
||||
resource: ceilometer_api_paste_ini
|
||||
ceilometer.conf:
|
||||
path: /etc/ceilometer/ceilometer.conf
|
||||
resource: ceilometer
|
||||
cinder-api-paste.ini:
|
||||
path: /etc/cinder/api-paste.ini
|
||||
resource: cinder_api_paste_ini
|
||||
cinder.conf:
|
||||
path: /etc/cinder/cinder.conf
|
||||
resource: cinder
|
||||
dhcp_agent.ini:
|
||||
path: /etc/neutron/dhcp_agent.ini
|
||||
resource: neutron_dhcp_agent_config
|
||||
glance-api.conf:
|
||||
path: /etc/glance/glance-api.conf
|
||||
resource: glance_api
|
||||
glance-cache.conf:
|
||||
path: /etc/glance/glance-cache.conf
|
||||
resource: glance_cache
|
||||
glance-glare.conf:
|
||||
path: /etc/glance/glance-glare.conf
|
||||
resource: glare_config
|
||||
glance-registry.conf:
|
||||
path: /etc/glance/glance-registry.conf
|
||||
resource: glance_registry
|
||||
heat-api-paste.ini:
|
||||
path: /etc/heat/api-paste.ini
|
||||
resource: heat_api_paste_ini
|
||||
heat.conf:
|
||||
path: /etc/heat/heat.conf
|
||||
resource: heat
|
||||
keystone.conf:
|
||||
path: /etc/keystone/keystone.conf
|
||||
resource: keystone_config
|
||||
l3_agent.ini:
|
||||
path: /etc/neutron/l3_agent.ini
|
||||
resource: neutron_l3_agent_config
|
||||
metadata_agent.ini:
|
||||
path: /etc/neutron/metadata_agent.ini
|
||||
resource: neutron_metadata_agent_config
|
||||
ml2_conf.ini:
|
||||
path: /etc/neutron/plugins/ml2/ml2_conf.ini
|
||||
resource: neutron_plugin_ml2
|
||||
neutron-api-paste.ini:
|
||||
path: /etc/neutron/api-paste.ini
|
||||
resource: neutron_api_config
|
||||
neutron.conf:
|
||||
path: /etc/neutron/neutron.conf
|
||||
resource: neutron_config
|
||||
nova-api-paste.ini:
|
||||
path: /etc/nova/api-paste.ini
|
||||
resource: nova_paste_api_ini
|
||||
nova.conf:
|
||||
path: /etc/nova/nova.conf
|
||||
resource: nova_config
|
||||
openvswitch_agent.ini:
|
||||
path: /etc/neutron/plugins/ml2/openvswitch_agent.ini
|
||||
resource: neutron_agent_ovs
|
||||
sriov_agent.ini:
|
||||
path: /etc/neutron/plugins/ml2/sriov_agent.ini
|
||||
resource: neutron_sriov_agent_config
|
||||
|
|
|
@ -18,12 +18,12 @@ from fuel_external_git import utils
|
|||
class TestUtils(base.TestCase):
|
||||
def test_extension_list(self):
|
||||
mapping = {
|
||||
'ceilometer_api_paste_ini': {
|
||||
'alias': 'ceilometer-api-paste.ini',
|
||||
'ceilometer-api-paste.ini': {
|
||||
'resource': 'ceilometer_api_paste_ini',
|
||||
'path': '/etc/ceilometer/api-paste.ini',
|
||||
},
|
||||
'ceilometer': {
|
||||
'alias': 'ceilometer.conf',
|
||||
'ceilometer.conf': {
|
||||
'resource': 'ceilometer',
|
||||
'path': '/etc/ceilometer/ceilometer.conf',
|
||||
}
|
||||
}
|
||||
|
|
|
@ -13,17 +13,11 @@
|
|||
import os
|
||||
from oslo_utils import importutils
|
||||
|
||||
from nailgun.db import db
|
||||
from nailgun.db.sqlalchemy.models import Cluster
|
||||
from nailgun.db.sqlalchemy.models import Release
|
||||
from nailgun.logger import logger
|
||||
|
||||
|
||||
def get_file_exts_list(resource_mapping):
|
||||
res = set()
|
||||
for resource in resource_mapping.values():
|
||||
res.add(resource['path'].split('.')[-1])
|
||||
return res
|
||||
return set(map(lambda key: key.split('.')[-1], resource_mapping.keys()))
|
||||
|
||||
|
||||
def get_config_hash(file_dir, resource_mapping, exts=['conf']):
|
||||
|
@ -33,23 +27,19 @@ def get_config_hash(file_dir, resource_mapping, exts=['conf']):
|
|||
"Directory {} not found. Returning emty dict".format(file_dir))
|
||||
return {}
|
||||
|
||||
cfg2drv = {}
|
||||
for resource, params in resource_mapping.items():
|
||||
drv = params.get(
|
||||
'driver',
|
||||
'fuel_external_git.drivers.openstack_config.OpenStackConfig'
|
||||
)
|
||||
cfg2drv[params['alias']] = {'drv': drv, 'resource': resource}
|
||||
|
||||
conf_files = [conf for conf in os.listdir(file_dir)
|
||||
if conf.split('.')[-1] in exts]
|
||||
|
||||
for conf_file in conf_files:
|
||||
if conf_file in cfg2drv.keys():
|
||||
drv_class = importutils.import_class(cfg2drv[conf_file]['drv'])
|
||||
if conf_file in resource_mapping.keys():
|
||||
drv = resource_mapping[conf_file].get(
|
||||
'driver',
|
||||
'fuel_external_git.drivers.openstack_config.OpenStackConfig'
|
||||
)
|
||||
drv_class = importutils.import_class(drv)
|
||||
config = drv_class(
|
||||
os.path.join(file_dir, conf_file),
|
||||
cfg2drv[conf_file]['resource']
|
||||
resource_mapping[conf_file]['resource']
|
||||
)
|
||||
res[config.config_name] = config.to_config_dict()
|
||||
return res
|
||||
|
@ -61,24 +51,3 @@ def deep_merge(dct, merge_dct):
|
|||
deep_merge(dct[k], merge_dct[k])
|
||||
else:
|
||||
dct[k] = merge_dct[k]
|
||||
|
||||
|
||||
# TODO(dukov) Remove this ugly staff once extension management is available
|
||||
def register_extension(ext_name):
|
||||
def decorator(cls):
|
||||
exts = {cl: cl['extensions'] for cl in
|
||||
list(db().query(Cluster).all()) +
|
||||
list(db().query(Release).all())}
|
||||
save = False
|
||||
for obj, extensions in exts.items():
|
||||
if u'fuel_external_git' not in extensions:
|
||||
extensions.append(u'fuel_external_git')
|
||||
extensions = list(set(extensions))
|
||||
obj.extensions = extensions
|
||||
save = True
|
||||
db().flush()
|
||||
|
||||
if save:
|
||||
db().commit()
|
||||
return cls
|
||||
return decorator
|
||||
|
|
Loading…
Reference in New Issue