Default data is loaded into memcached by processor
Closes bug 1307447 Change-Id: I2347a7addfa18f76a37cbac2d7cf49291a30f63a
This commit is contained in:
Ilya Shakhat
parent
b6af7136d1
commit
e1cfa40552
|
|
@ -88,7 +88,7 @@ def exception_handler():
|
|||
if isinstance(e, exceptions.HTTPException):
|
||||
raise # ignore Flask exceptions
|
||||
LOG.exception(e)
|
||||
flask.abort(404)
|
||||
flask.abort(500)
|
||||
|
||||
return exception_handler_decorated_function
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ import memcache
|
|||
|
||||
from driverlog.dashboard import memory_storage
|
||||
from driverlog.openstack.common import log as logging
|
||||
from driverlog.processor import utils
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
|
@ -105,24 +104,10 @@ def get_vault():
|
|||
flask.abort(500)
|
||||
|
||||
conf = flask.current_app.config['CONF']
|
||||
dd_uri = conf.default_data_uri
|
||||
vault['default_data'] = utils.read_json_from_uri(dd_uri)
|
||||
|
||||
if not vault['default_data']:
|
||||
LOG.critical('Default data config file "%s" is not found',
|
||||
dd_uri)
|
||||
flask.abort(500)
|
||||
|
||||
levels_map = _build_levels_map()
|
||||
vault['levels_map'] = levels_map
|
||||
|
||||
projects_map = _build_projects_map(vault['default_data'])
|
||||
vault['projects_map'] = projects_map
|
||||
|
||||
drivers_map = _build_drivers_map(vault['default_data'], levels_map,
|
||||
projects_map)
|
||||
vault['drivers_map'] = drivers_map
|
||||
|
||||
MEMCACHED_URI_PREFIX = r'^memcached:\/\/'
|
||||
stripped = re.sub(MEMCACHED_URI_PREFIX, '',
|
||||
conf.runtime_storage_uri)
|
||||
|
|
@ -141,8 +126,24 @@ def get_vault():
|
|||
flask.request.driverlog_updated = True
|
||||
|
||||
memcached = vault['memcached']
|
||||
update = memcached.get('driverlog:update')
|
||||
if update:
|
||||
hashes = memcached.get_multi(['default_data_hash', 'update_hash'],
|
||||
key_prefix='driverlog:')
|
||||
|
||||
if vault.get('default_data_hash') != hashes.get('default_data_hash'):
|
||||
vault['default_data_hash'] = hashes['default_data_hash']
|
||||
vault['default_data'] = memcached.get('driverlog:default_data')
|
||||
|
||||
projects_map = _build_projects_map(vault['default_data'])
|
||||
vault['projects_map'] = projects_map
|
||||
|
||||
drivers_map = _build_drivers_map(
|
||||
vault['default_data'], vault['levels_map'], projects_map)
|
||||
vault['drivers_map'] = drivers_map
|
||||
|
||||
if vault.get('update_hash') != hashes.get('update_hash'):
|
||||
vault['update_hash'] = hashes['update_hash']
|
||||
update = memcached.get('driverlog:update')
|
||||
|
||||
levels_map = vault['levels_map']
|
||||
|
||||
for proj_vendor_driver, os_versions_map in update.iteritems():
|
||||
|
|
@ -159,6 +160,10 @@ def get_vault():
|
|||
vault['drivers_map'][proj_vendor_driver][
|
||||
'os_versions_map'].update(ovm)
|
||||
|
||||
if not vault.get('default_data'):
|
||||
raise Exception('Memcached is not initialized. '
|
||||
'Please run the processor')
|
||||
|
||||
return vault
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ else:
|
|||
# Handlers ---------
|
||||
|
||||
@app.route('/')
|
||||
@decorators.exception_handler()
|
||||
@decorators.templated()
|
||||
def summary():
|
||||
selected_project_id = parameters.get_single_parameter({}, 'project_id')
|
||||
|
|
|
|||
|
|
@ -12,12 +12,15 @@
|
|||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import re
|
||||
|
||||
import memcache
|
||||
from oslo.config import cfg
|
||||
from six.moves.urllib import parse
|
||||
import time
|
||||
|
||||
from driverlog.openstack.common import log as logging
|
||||
from driverlog.processor import config
|
||||
|
|
@ -38,7 +41,7 @@ def update_generator(memcached, default_data, ci_ids_map, force_update=False):
|
|||
|
||||
LOG.debug('Processing reviews for project: %s', project_id)
|
||||
|
||||
rcs_key = 'rcs:' + parse.quote_plus(project_id)
|
||||
rcs_key = 'driverlog:rcs:' + parse.quote_plus(project_id)
|
||||
last_id = None
|
||||
if not force_update:
|
||||
last_id = memcached.get(rcs_key)
|
||||
|
|
@ -100,6 +103,12 @@ def update_generator(memcached, default_data, ci_ids_map, force_update=False):
|
|||
memcached.set(rcs_key, last_id)
|
||||
|
||||
|
||||
def _get_hash(data):
|
||||
h = hashlib.new('sha1')
|
||||
h.update(json.dumps(data))
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def main():
|
||||
# init conf and logging
|
||||
conf = cfg.CONF
|
||||
|
|
@ -116,7 +125,7 @@ def main():
|
|||
exit(1)
|
||||
|
||||
memcached_uri = stripped.split(',')
|
||||
memcached = memcache.Client(memcached_uri)
|
||||
memcache_inst = memcache.Client(memcached_uri)
|
||||
|
||||
default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri)
|
||||
if not default_data:
|
||||
|
|
@ -132,26 +141,33 @@ def main():
|
|||
ci_id = os_version['ci_id']
|
||||
ci_ids_map[ci_id] = (vendor, driver_name)
|
||||
|
||||
persisted_data = {}
|
||||
update = {}
|
||||
if not cfg.CONF.force_update:
|
||||
persisted_data = memcached.get('driverlog:update') or {}
|
||||
update = memcache_inst.get('driverlog:update') or {}
|
||||
|
||||
for record in update_generator(memcached, default_data, ci_ids_map,
|
||||
has_update = False
|
||||
|
||||
for record in update_generator(memcache_inst, default_data, ci_ids_map,
|
||||
force_update=cfg.CONF.force_update):
|
||||
LOG.info('Got new record from Gerrit: %s', record)
|
||||
has_update = True
|
||||
|
||||
key = record.keys()[0]
|
||||
if key not in persisted_data:
|
||||
persisted_data.update(record)
|
||||
if key not in update:
|
||||
update.update(record)
|
||||
else:
|
||||
persisted_os_versions = persisted_data[key]['os_versions_map']
|
||||
persisted_os_versions = update[key]['os_versions_map']
|
||||
for os_version, info in record[key]['os_versions_map'].iteritems():
|
||||
if os_version not in persisted_os_versions:
|
||||
persisted_os_versions[os_version] = info
|
||||
else:
|
||||
persisted_os_versions[os_version].update(info)
|
||||
|
||||
memcached.set('driverlog:update', persisted_data)
|
||||
memcache_inst.set('driverlog:default_data', default_data)
|
||||
memcache_inst.set('driverlog:update', update)
|
||||
memcache_inst.set('driverlog:default_data_hash', _get_hash(default_data))
|
||||
if has_update:
|
||||
memcache_inst.set('driverlog:update_hash', time.time())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
Loading…
Reference in New Issue