Extension performance improvement

* Introduced Git repo stat TTL. This helps to avoid unnecessary fetch
  requests to remote repository
* Configuration file restructured. It's easier to have mapping for file
  name in Git repository on resource rather then vise versa

Change-Id: I222199fc8aa203966a465508d4ecd8477f0fac3e
This commit is contained in:
Dmitry Ukov 2016-09-27 17:06:22 +03:00
parent 40e1ede7b9
commit 0e58b576b3
7 changed files with 107 additions and 128 deletions

View File

@ -11,3 +11,4 @@
# under the License.
REPOS_DIR = '/var/lib/fuel_repos'
REPO_TTL = 1200

View File

@ -33,22 +33,12 @@ class OpenStackConfigPipeline(BasePipeline):
Genereate OpenStack configuration hash based on configuration files
stored in git repository associated with a particular environment
Example of configuration extension:
configuration:
nova_config:
DEFAULT/debug:
value: True
DEFAULT/amqp_durable_queues:
value: False
keystone_config:
DEFAULT/default_publisher_id:
ensure: absent
DEFAULT/crypt_strength:
value: 6000
"""
logger.info("Started serialisation for node {}".format(node.id))
repo = GitRepo.get_by_cluster_id(node.cluster_id)
if not repo:
return node_data
GitRepo.checkout(repo)
repo_path = os.path.join(const.REPOS_DIR, repo.repo_name)
resource_mapping = ExternalGit.ext_settings['resource_mapping']
@ -99,14 +89,17 @@ class OpenStackConfigPipeline(BasePipeline):
override_configs['nodes'].get(uid, {}))
node_data['configuration'] = common
logger.info("Node {0} config from git {1}".format(uid, common))
logger.debug("Node {0} config from git {1}".format(uid, common))
logger.info("Finished serialisation for node {}".format(node.id))
return node_data
@classmethod
def process_deployment_for_cluster(self, cluster, data):
logger.info("Started serialisation for cluster {}".format(cluster.id))
repo = GitRepo.get_by_cluster_id(cluster.id)
if not repo:
return data
if repo.manage_master:
GitRepo.checkout(repo)
repo_path = os.path.join(const.REPOS_DIR, repo.repo_name)
@ -118,11 +111,11 @@ class OpenStackConfigPipeline(BasePipeline):
)
data['master_config'] = master_config
logger.info("Finished serialisation for cluster {}".format(cluster.id))
return data
# TODO(dukov) Remove decorator extension management is available
@utils.register_extension(u'fuel_external_git')
class ExternalGit(BaseExtension):
name = 'fuel_external_git'
version = '1.0.0'

View File

@ -12,8 +12,8 @@
single_schema = {
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "Cluster",
"description": "Serialized Cluster object",
"title": "GitRepo",
"description": "Serialized GitRepo object",
"type": "object",
"properties": {
"id": {"type": "number"},

View File

@ -12,6 +12,7 @@
import os
import shutil
import time
import yaml
from distutils.dir_util import copy_tree
@ -22,6 +23,7 @@ from fuel_external_git.models import GitRepo
from git import exc
from git import Repo
from nailgun.consts import CLUSTER_STATUSES
from nailgun.db import db
from nailgun import errors
from nailgun.logger import logger
@ -92,6 +94,20 @@ class GitRepo(NailgunObject):
@classmethod
def checkout(self, instance):
fetch_file = os.path.join(
const.REPOS_DIR,
instance.repo_name,
'.git/FETCH_HEAD'
)
if os.path.exists(fetch_file):
current_ts = time.time()
cluster = Cluster.get_by_uid(instance.env_id)
last_fetch = os.stat(fetch_file).st_mtime
if cluster.status != CLUSTER_STATUSES.deployment and \
current_ts - last_fetch < const.REPO_TTL:
return
logger.debug("Repo TTL exceeded. Fetching code...")
ssh_cmd = self._get_ssh_cmd(instance.repo_name)
if not os.path.exists(self._get_key_path(instance.repo_name)):

View File

@ -1,69 +1,69 @@
resource_mapping:
ceilometer_api_paste_ini:
alias: ceilometer-api-paste.ini
path: /etc/ceilometer/api-paste.ini
ceilometer:
alias: ceilometer.conf
path: /etc/ceilometer/ceilometer.conf
cinder_api_paste_ini:
alias: cinder-api-paste.ini
path: /etc/cinder/api-paste.ini
cinder:
alias: cinder.conf
path: /etc/cinder/cinder.conf
glance_api:
alias: glance-api.conf
path: /etc/glance/glance-api.conf
glance_cache:
alias: glance-cache.conf
path: /etc/glance/glance-cache.conf
glare_config:
alias: glance-glare.conf
path: /etc/glance/glance-glare.conf
glance_registry:
alias: glance-registry.conf
path: /etc/glance/glance-registry.conf
heat_api_paste_ini:
alias: heat-api-paste.ini
path: /etc/heat/api-paste.ini
heat:
alias: heat.conf
path: /etc/heat/heat.conf
keystone_config:
alias: keystone.conf
path: /etc/keystone/keystone.conf
neutron_agent_ovs:
alias: openvswitch_agent.ini
path: /etc/neutron/plugins/ml2/openvswitch_agent.ini
neutron_api_config:
alias: neutron-api-paste.ini
path: /etc/neutron/api-paste.ini
neutron_config:
alias: neutron.conf
path: /etc/neutron/neutron.conf
neutron_dhcp_agent_config:
alias: dhcp_agent.ini
path: /etc/neutron/dhcp_agent.ini
neutron_l3_agent_config:
alias: l3_agent.ini
path: /etc/neutron/l3_agent.ini
neutron_metadata_agent_config:
alias: metadata_agent.ini
path: /etc/neutron/metadata_agent.ini
neutron_plugin_ml2:
alias: ml2_conf.ini
path: /etc/neutron/plugins/ml2/ml2_conf.ini
neutron_sriov_agent_config:
alias: sriov_agent.ini
path: /etc/neutron/plugins/ml2/sriov_agent.ini
nova_config:
alias: nova.conf
path: /etc/nova/nova.conf
nova_paste_api_ini:
alias: nova-api-paste.ini
path: /etc/nova/api-paste.ini
master_mapping:
master_config:
alias: master_config.yaml
path: ""
driver: 'fuel_external_git.drivers.yaml_driver.YamlConfig'
master_config.yaml:
resource: master_config
driver: fuel_external_git.drivers.yaml_driver.YamlConfig
path: ''
resource_mapping:
ceilometer-api-paste.ini:
path: /etc/ceilometer/api-paste.ini
resource: ceilometer_api_paste_ini
ceilometer.conf:
path: /etc/ceilometer/ceilometer.conf
resource: ceilometer
cinder-api-paste.ini:
path: /etc/cinder/api-paste.ini
resource: cinder_api_paste_ini
cinder.conf:
path: /etc/cinder/cinder.conf
resource: cinder
dhcp_agent.ini:
path: /etc/neutron/dhcp_agent.ini
resource: neutron_dhcp_agent_config
glance-api.conf:
path: /etc/glance/glance-api.conf
resource: glance_api
glance-cache.conf:
path: /etc/glance/glance-cache.conf
resource: glance_cache
glance-glare.conf:
path: /etc/glance/glance-glare.conf
resource: glare_config
glance-registry.conf:
path: /etc/glance/glance-registry.conf
resource: glance_registry
heat-api-paste.ini:
path: /etc/heat/api-paste.ini
resource: heat_api_paste_ini
heat.conf:
path: /etc/heat/heat.conf
resource: heat
keystone.conf:
path: /etc/keystone/keystone.conf
resource: keystone_config
l3_agent.ini:
path: /etc/neutron/l3_agent.ini
resource: neutron_l3_agent_config
metadata_agent.ini:
path: /etc/neutron/metadata_agent.ini
resource: neutron_metadata_agent_config
ml2_conf.ini:
path: /etc/neutron/plugins/ml2/ml2_conf.ini
resource: neutron_plugin_ml2
neutron-api-paste.ini:
path: /etc/neutron/api-paste.ini
resource: neutron_api_config
neutron.conf:
path: /etc/neutron/neutron.conf
resource: neutron_config
nova-api-paste.ini:
path: /etc/nova/api-paste.ini
resource: nova_paste_api_ini
nova.conf:
path: /etc/nova/nova.conf
resource: nova_config
openvswitch_agent.ini:
path: /etc/neutron/plugins/ml2/openvswitch_agent.ini
resource: neutron_agent_ovs
sriov_agent.ini:
path: /etc/neutron/plugins/ml2/sriov_agent.ini
resource: neutron_sriov_agent_config

View File

@ -18,12 +18,12 @@ from fuel_external_git import utils
class TestUtils(base.TestCase):
def test_extension_list(self):
mapping = {
'ceilometer_api_paste_ini': {
'alias': 'ceilometer-api-paste.ini',
'ceilometer-api-paste.ini': {
'resource': 'ceilometer_api_paste_ini',
'path': '/etc/ceilometer/api-paste.ini',
},
'ceilometer': {
'alias': 'ceilometer.conf',
'ceilometer.conf': {
'resource': 'ceilometer',
'path': '/etc/ceilometer/ceilometer.conf',
}
}

View File

@ -13,17 +13,11 @@
import os
from oslo_utils import importutils
from nailgun.db import db
from nailgun.db.sqlalchemy.models import Cluster
from nailgun.db.sqlalchemy.models import Release
from nailgun.logger import logger
def get_file_exts_list(resource_mapping):
res = set()
for resource in resource_mapping.values():
res.add(resource['path'].split('.')[-1])
return res
return set(map(lambda key: key.split('.')[-1], resource_mapping.keys()))
def get_config_hash(file_dir, resource_mapping, exts=['conf']):
@ -33,23 +27,19 @@ def get_config_hash(file_dir, resource_mapping, exts=['conf']):
"Directory {} not found. Returning emty dict".format(file_dir))
return {}
cfg2drv = {}
for resource, params in resource_mapping.items():
drv = params.get(
'driver',
'fuel_external_git.drivers.openstack_config.OpenStackConfig'
)
cfg2drv[params['alias']] = {'drv': drv, 'resource': resource}
conf_files = [conf for conf in os.listdir(file_dir)
if conf.split('.')[-1] in exts]
for conf_file in conf_files:
if conf_file in cfg2drv.keys():
drv_class = importutils.import_class(cfg2drv[conf_file]['drv'])
if conf_file in resource_mapping.keys():
drv = resource_mapping[conf_file].get(
'driver',
'fuel_external_git.drivers.openstack_config.OpenStackConfig'
)
drv_class = importutils.import_class(drv)
config = drv_class(
os.path.join(file_dir, conf_file),
cfg2drv[conf_file]['resource']
resource_mapping[conf_file]['resource']
)
res[config.config_name] = config.to_config_dict()
return res
@ -61,24 +51,3 @@ def deep_merge(dct, merge_dct):
deep_merge(dct[k], merge_dct[k])
else:
dct[k] = merge_dct[k]
# TODO(dukov) Remove this ugly staff once extension management is available
def register_extension(ext_name):
def decorator(cls):
exts = {cl: cl['extensions'] for cl in
list(db().query(Cluster).all()) +
list(db().query(Release).all())}
save = False
for obj, extensions in exts.items():
if u'fuel_external_git' not in extensions:
extensions.append(u'fuel_external_git')
extensions = list(set(extensions))
obj.extensions = extensions
save = True
db().flush()
if save:
db().commit()
return cls
return decorator