openstack
/
freezer-api
Code Issues Proposed changes

Support search otion and limit, offset for sqlalchemy 

Change-Id: Ieeaff66e865cf4222b66ca6bb3677c579909bf16
This commit is contained in:
gengchc2 2018-12-03 19:17:16 -08:00
parent 7de745fed1
commit a73b804376
1 changed files with 64 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
freezer_api/db/sqlalchemy/api.py
View File

@ -256,20 +256,24 @@ def replace_tuple(tablename, user_id, tuple_id, tuple_values, project_id=None):
def search_tuple(tablename, user_id, project_id=None, offset=0, def search_tuple(tablename, user_id, project_id=None, offset=0,
limit=100, search=None): limit=100, search=None):
search = valid_and_get_search_option(search=search)
session = get_db_session() session = get_db_session()
with session.begin(): with session.begin():
try: try:
# TODO(gecong) search will be implemented in the future # TODO(gecong) search will be implemented in the future
query = model_query(session, tablename, project_id=project_id) query = model_query(session, tablename, project_id=project_id)
query = query.filter_by(user_id=user_id) query = query.filter_by(user_id=user_id)
query = query.offset(offset) # If search option isn't valid or set, we use limit and offset
query = query.limit(limit) # in sqlalchemy level
if len(search) == 0:
query = query.offset(offset)
query = query.limit(limit)
result = query.all() result = query.all()
except Exception as e: except Exception as e:
raise freezer_api_exc.StorageEngineError( raise freezer_api_exc.StorageEngineError(
message='Mysql operation failed {0}'.format(e)) message='Mysql operation failed {0}'.format(e))
session.close() session.close()
return result return result, search
def get_recursively(source_dict, search_keys): def get_recursively(source_dict, search_keys):
@ -318,28 +322,24 @@ def valid_and_get_search_option(search=None):
except Exception as e: except Exception as e:
msg = "{0} \n json_decode error: {1}".\ msg = "{0} \n json_decode error: {1}".\
format(error_msg, e) format(error_msg, e)
LOG.error(error_msg) LOG.error(msg)
raise Exception(msg) return {}
if not isinstance(search, list): if not isinstance(search, list):
LOG.error(error_msg) LOG.error(error_msg)
raise Exception(error_msg) return {}
search = {'match': search} search = {'match': search}
if not isinstance(search, dict): if not isinstance(search, dict):
search = {} return {}
return search return search
def filter_tuple_by_search_opt(tuples, search=None): def filter_tuple_by_search_opt(tuples, offset=0, limit=100, search=None):
search = search or {} search = search or {}
search_key = {}
result_last = []
# search opt is null, all tuples will be filtered in. # search opt is null, all tuples will be filtered in.
if len(search) == 0: if len(search) == 0:
return tuples return tuples
search = valid_and_get_search_option(search=search)
if len(search) == 0:
return tuples
result_last = []
search_key = {}
for m in search.get('match', []): for m in search.get('match', []):
for key, value in m.items(): for key, value in m.items():
search_key[key] = value search_key[key] = value
@ -347,7 +347,11 @@ def filter_tuple_by_search_opt(tuples, search=None):
for key, value in m.items(): for key, value in m.items():
search_key[key] = value search_key[key] = value
jobs_search_offset = 0
jobs_search_count = 0
for tuple in tuples: for tuple in tuples:
if jobs_search_count >= limit:
return result_last
filter_out = False filter_out = False
search_keys_found = get_recursively(tuple, search_key) search_keys_found = get_recursively(tuple, search_key)
# If all keys and values are in search_keys_found, this tuple will be # If all keys and values are in search_keys_found, this tuple will be
@ -365,14 +369,15 @@ def filter_tuple_by_search_opt(tuples, search=None):
filter_out = True filter_out = True
break break
if not filter_out: if not filter_out:
result_last.append(tuple) jobs_search_offset += 1
if jobs_search_offset > offset:
jobs_search_count += 1
result_last.append(tuple)
return result_last return result_last
def get_client(user_id, project_id=None, client_id=None, offset=0, def get_client_byid(user_id, client_id, project_id=None):
limit=100, search=None):
clients = []
session = get_db_session() session = get_db_session()
with session.begin(): with session.begin():
try: try:
@ -380,15 +385,26 @@ def get_client(user_id, project_id=None, client_id=None, offset=0,
if client_id: if client_id:
query = query.filter_by(user_id=user_id).filter_by( query = query.filter_by(user_id=user_id).filter_by(
client_id=client_id) client_id=client_id)
else: result = query.all()
query = query.filter_by(user_id=user_id)
query = query.offset(offset)
query = query.limit(limit)
result = query.all()
except Exception as e: except Exception as e:
raise freezer_api_exc.StorageEngineError( raise freezer_api_exc.StorageEngineError(
message='Mysql operation failed {0}'.format(e)) message='Mysql operation failed {0}'.format(e))
session.close() session.close()
return result
def get_client(user_id, project_id=None, client_id=None, offset=0,
limit=100, search=None):
clients = []
search_key = {}
if client_id:
result = get_client_byid(user_id, client_id, project_id=project_id)
else:
result, search_key = search_tuple(tablename=models.Client,
user_id=user_id,
project_id=project_id, offset=offset,
limit=limit, search=search)
for client in result: for client in result:
clientmap = {} clientmap = {}
@ -402,11 +418,9 @@ def get_client(user_id, project_id=None, client_id=None, offset=0,
# If search opt is wrong, filter will not work, # If search opt is wrong, filter will not work,
# return all tuples. # return all tuples.
try: if not client_id:
clients = filter_tuple_by_search_opt(clients, search) clients = filter_tuple_by_search_opt(clients, offset=offset,
except Exception as e: limit=limit, search=search_key)
LOG.error(e)
return clients return clients
@ -564,9 +578,9 @@ def search_action(user_id, project_id=None, offset=0,
actions = [] actions = []
result = search_tuple(tablename=models.Action, user_id=user_id, result, search_key = search_tuple(tablename=models.Action, user_id=user_id,
project_id=project_id, offset=offset, project_id=project_id, offset=offset,
limit=limit) limit=limit, search=search)
for action in result: for action in result:
actionmap = {} actionmap = {}
actionmap['project_id'] = project_id actionmap['project_id'] = project_id
@ -593,10 +607,8 @@ def search_action(user_id, project_id=None, offset=0,
actions.append(actionmap) actions.append(actionmap)
# If search opt is wrong, filter will not work, # If search opt is wrong, filter will not work,
# return all tuples. # return all tuples.
try: actions = filter_tuple_by_search_opt(actions, offset=offset, limit=limit,
actions = filter_tuple_by_search_opt(actions, search) search=search_key)
except Exception as e:
LOG.error(e)
return actions return actions
@ -732,9 +744,9 @@ def get_job(user_id, job_id, project_id=None):
def search_job(user_id, project_id=None, offset=0, def search_job(user_id, project_id=None, offset=0,
limit=100, search=None): limit=100, search=None):
jobs = [] jobs = []
result = search_tuple(tablename=models.Job, user_id=user_id, result, search_key = search_tuple(tablename=models.Job, user_id=user_id,
project_id=project_id, offset=offset, project_id=project_id, offset=offset,
limit=limit) limit=limit, search=search)
for job in result: for job in result:
jobmap = {} jobmap = {}
jobmap['job_id'] = job.get('id') jobmap['job_id'] = job.get('id')
@ -751,10 +763,8 @@ def search_job(user_id, project_id=None, offset=0,
jobs.append(jobmap) jobs.append(jobmap)
# If search opt is wrong, filter will not work, # If search opt is wrong, filter will not work,
# return all tuples. # return all tuples.
try: jobs = filter_tuple_by_search_opt(jobs, offset=offset, limit=limit,
jobs = filter_tuple_by_search_opt(jobs, search) search=search_key)
except Exception as e:
LOG.error(e)
return jobs return jobs
@ -875,9 +885,9 @@ def search_backup(user_id, project_id=None, offset=0,
limit=100, search=None): limit=100, search=None):
backups = [] backups = []
result = search_tuple(tablename=models.Backup, user_id=user_id, result, search_key = search_tuple(tablename=models.Backup, user_id=user_id,
project_id=project_id, offset=offset, project_id=project_id, offset=offset,
limit=limit) limit=limit, search=search)
for backup in result: for backup in result:
backupmap = {} backupmap = {}
backupmap['project_id'] = project_id backupmap['project_id'] = project_id
@ -887,14 +897,10 @@ def search_backup(user_id, project_id=None, offset=0,
backupmap['backup_metadata'] = json_utils.\ backupmap['backup_metadata'] = json_utils.\
json_decode(backup.get('backup_metadata')) json_decode(backup.get('backup_metadata'))
backups.append(backupmap) backups.append(backupmap)
# If search opt is wrong, filter will not work, # If search opt is wrong, filter will not work,
# return all tuples. # return all tuples.
try: backups = filter_tuple_by_search_opt(backups, offset=offset, limit=limit,
backups = filter_tuple_by_search_opt(backups, search) search=search_key)
except Exception as e:
LOG.error(e)
return backups return backups
@ -1052,9 +1058,10 @@ def search_session(user_id, project_id=None, offset=0,
sessions = [] sessions = []
jobt = {} jobt = {}
result = search_tuple(tablename=models.Session, user_id=user_id, result, search_key = search_tuple(tablename=models.Session,
project_id=project_id, offset=offset, user_id=user_id,
limit=limit) project_id=project_id, offset=offset,
limit=limit, search=search)
for sessiont in result: for sessiont in result:
sessionmap = {} sessionmap = {}
sessionmap['project_id'] = project_id sessionmap['project_id'] = project_id
@ -1077,8 +1084,7 @@ def search_session(user_id, project_id=None, offset=0,
sessions.append(sessionmap) sessions.append(sessionmap)
# If search opt is wrong, filter will not work, # If search opt is wrong, filter will not work,
# return all tuples. # return all tuples.
try: sessions = filter_tuple_by_search_opt(sessions, offset=offset,
sessions = filter_tuple_by_search_opt(sessions, search) limit=limit, search=search_key)
except Exception as e:
LOG.error(e)
return sessions return sessions