openstack
/
storlets
Code Issues Proposed changes

Merge "Refactor SLO tests"

This commit is contained in:
Zuul 2024-01-29 06:16:20 +00:00 committed by Gerrit Code Review
parent dc90c16808 50a4d4f73a
commit 3eb72a4bda
2 changed files with 69 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
tests/functional/java/test_SLO.py
View File

@ -15,142 +15,78 @@
# limitations under the License. # limitations under the License.
import json import json
import os
import random import random
import string import string
from swiftclient import client as c from swiftclient import client
from tests.functional.java import StorletJavaFunctionalTest from tests.functional.java import StorletJavaFunctionalTest
import unittest import unittest
def create_local_chunks():
for i in range(1, 10):
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'wb')
f.write(b''.join(
random.choice(
string.ascii_uppercase + string.digits).encode("utf-8")
for _ in range(1048576)))
f.close()
def delete_local_chunks():
for i in range(1, 10):
oname = '/tmp/slo_chunk_%d' % i
os.remove(oname)
class TestSLO(StorletJavaFunctionalTest): class TestSLO(StorletJavaFunctionalTest):
def setUp(self): def setUp(self):
self.storlet_log = 'identitystorlet-1.0.log' self.storlet_log = 'identitystorlet-1.0.log'
self.additional_headers = {} self.additional_headers = {}
self.chunks = []
main_class = 'org.openstack.storlet.identity.IdentityStorlet' main_class = 'org.openstack.storlet.identity.IdentityStorlet'
super(TestSLO, self).setUp('IdentityStorlet', super(TestSLO, self).setUp('IdentityStorlet',
'identitystorlet-1.0.jar', 'identitystorlet-1.0.jar',
main_class, main_class,
'') '')
for cont in ('container1', 'container2', 'container3'): self.create_local_chunks()
self.create_container(cont)
create_local_chunks()
self.put_SLO() self.put_SLO()
self.get_SLO()
def tearDown(self): def create_local_chunks(self):
delete_local_chunks() for i in range(10):
super(TestSLO, self).tearDown() self.chunks.append(
''.join([random.choice(string.ascii_uppercase + string.digits)
def get_SLO(self): for _ in range(1024)]).encode('ascii'))
response = dict()
headers, body = c.get_object(self.url, self.token,
self.container, 'assembly',
http_conn=None, resp_chunk_size=1048576,
query_string=None, response_dict=response,
headers=None)
i = 1
for chunk in body:
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'rb')
file_content = f.read()
# print '%s %s' % (chunk[:10], file_content[:10])
# print '%d %d' % (len(chunk), len(file_content))
self.assertEqual(chunk, file_content)
f.close()
i = i + 1
def put_SLO(self): def put_SLO(self):
# Create temp files
assembly = [] assembly = []
for i in range(1, 10): for i in range(10):
oname = '/tmp/slo_chunk_%d' % i oname = 'slo_chunk_%d' % i
f = open(oname, 'rb') etag = client.put_object(self.url, self.token,
content_length = None self.container, oname, self.chunks[i],
response = dict() content_type="application/octet-stream")
c.put_object(self.url, self.token,
self.container, oname, f,
content_length, None, None,
"application/octet-stream",
None, None, None, None, response)
f.close()
status = response.get('status')
self.assertGreaterEqual(status, 200)
self.assertLess(status, 300)
headers = response.get('headers')
segment = dict() segment = dict()
segment['path'] = '%s/%s' % (self.container, oname) segment['path'] = '%s/%s' % (self.container, oname)
segment['size_bytes'] = 1048576 segment['size_bytes'] = 1024
segment['etag'] = headers['etag'] segment['etag'] = etag
assembly.append(segment) assembly.append(segment)
content_length = None
response = dict()
headers = {'x-object-meta-prop1': 'val1'} headers = {'x-object-meta-prop1': 'val1'}
c.put_object(self.url, self.token, self.container, client.put_object(self.url, self.token, self.container,
'assembly', json.dumps(assembly), 'assembly', json.dumps(assembly),
content_length=None, etag=None, chunk_size=None, headers=headers,
headers=headers, query_string='multipart-manifest=put', query_string='multipart-manifest=put')
response_dict=response)
status = response.get('status')
self.assertGreaterEqual(status, 200)
self.assertLess(status, 300)
def compare_slo_to_chunks(self, body): def compare_slo_to_chunks(self, body):
i = 1 length = 0
for chunk in body: for (i, chunk) in enumerate(body):
if chunk: self.assertEqual(chunk, self.chunks[i])
if i < 10: length += 1
oname = '/tmp/slo_chunk_%d' % i self.assertEqual(length, 10)
f = open(oname, 'rb')
file_content = f.read()
# print '%s %s' % (chunk[:10], file_content[:10])
# print '%d %d' % (len(chunk), len(file_content))
self.assertEqual(chunk, file_content)
f.close()
i = i + 1
else:
aux_content = ''
for j in range(1, 4):
oname = '/tmp/aux_file%d' % j
f = open(oname, 'rb')
aux_content += f.read()
f.close()
self.ssertEqual(chunk, aux_content)
def test_get_SLO(self): def test_get_SLO_without_storlet(self):
headers = {'X-Run-Storlet': self.storlet_name} _, body = client.get_object(self.url, self.token,
headers.update(self.additional_headers) self.container, 'assembly',
response = dict() resp_chunk_size=1024)
headers, body = c.get_object(self.url, self.token,
self.container, 'assembly',
query_string=None,
response_dict=response,
resp_chunk_size=1048576,
headers=headers)
self.compare_slo_to_chunks(body) self.compare_slo_to_chunks(body)
test_get_SLO.slo = 1 test_get_SLO_without_storlet.slow = 1
def test_get_SLO_with_storlet(self):
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
_, body = client.get_object(self.url, self.token,
self.container, 'assembly',
resp_chunk_size=1024,
headers=headers)
self.compare_slo_to_chunks(body)
test_get_SLO_with_storlet.slow = 1
class TestSloOnProxy(TestSLO): class TestSloOnProxy(TestSLO):

87
tests/functional/python/test_SLO.py
View File

@ -33,89 +33,60 @@ class TestSLO(StorletPythonFunctionalTest):
storlet_file=None, storlet_file=None,
headers={}) headers={})
for cont in ('container1', 'container2', 'container3'):
self.create_container(cont)
self.create_local_chunks() self.create_local_chunks()
self.put_SLO() self.put_SLO()
self.get_SLO()
def create_local_chunks(self): def create_local_chunks(self):
for i in range(9): for i in range(10):
self.chunks.append( self.chunks.append(
''.join([random.choice(string.ascii_uppercase + string.digits) ''.join([random.choice(string.ascii_uppercase + string.digits)
for _ in range(1024 * 1024)]).encode('ascii')) for _ in range(1024)]).encode('ascii'))
def get_SLO(self):
response = dict()
headers, body = client.get_object(self.url, self.token, self.container,
'assembly', http_conn=None,
resp_chunk_size=1024 * 1024,
query_string=None,
response_dict=response,
headers=None)
for (i, chunk) in enumerate(body):
self.assertEqual(chunk, self.chunks[i])
def put_SLO(self): def put_SLO(self):
assembly = [] assembly = []
for i in range(9): for i in range(10):
oname = 'slo_chunk_%d' % i oname = 'slo_chunk_%d' % i
content_length = None etag = client.put_object(self.url, self.token,
response = dict() self.container, oname, self.chunks[i],
client.put_object(self.url, self.token, content_type="application/octet-stream")
self.container, oname, self.chunks[i],
content_length, None, None,
"application/octet-stream",
None, None, None, None, response)
status = response.get('status')
self.assertEqual(2, status // 100)
headers = response.get('headers')
segment = dict() segment = dict()
segment['path'] = '%s/%s' % (self.container, oname) segment['path'] = '%s/%s' % (self.container, oname)
segment['size_bytes'] = 1024 * 1024 segment['size_bytes'] = 1024
segment['etag'] = headers['etag'] segment['etag'] = etag
assembly.append(segment) assembly.append(segment)
content_length = None
response = dict()
headers = {'x-object-meta-prop1': 'val1'} headers = {'x-object-meta-prop1': 'val1'}
client.put_object(self.url, self.token, self.container, client.put_object(self.url, self.token, self.container,
'assembly', json.dumps(assembly), 'assembly', json.dumps(assembly),
content_length=None, etag=None, chunk_size=None,
headers=headers, headers=headers,
query_string='multipart-manifest=put', query_string='multipart-manifest=put')
response_dict=response)
status = response.get('status')
self.assertEqual(2, status // 100)
def compare_slo_to_chunks(self, body): def compare_slo_to_chunks(self, body):
length = 0
for (i, chunk) in enumerate(body): for (i, chunk) in enumerate(body):
if chunk: self.assertEqual(chunk, self.chunks[i])
if i in range(9): length += 1
self.assertEqual(chunk, self.chunks[i]) self.assertEqual(length, 10)
else:
aux_content = b''
for j in range(1, 4):
oname = 'aux_file%d' % j
with open(oname, 'rb') as f:
aux_content += f.read()
self.asertEqual(chunk, aux_content)
def test_get_SLO(self): def test_get_SLO_without_storlet(self):
headers = {'X-Run-Storlet': self.storlet_name} _, body = client.get_object(self.url, self.token,
headers.update(self.additional_headers) self.container, 'assembly',
response = dict() resp_chunk_size=1024)
headers, body = client.get_object(self.url, self.token,
self.container, 'assembly',
query_string=None,
response_dict=response,
resp_chunk_size=1024 * 1024,
headers=headers)
self.compare_slo_to_chunks(body) self.compare_slo_to_chunks(body)
test_get_SLO.slow = 1 test_get_SLO_without_storlet.slow = 1
def test_get_SLO_with_storlet(self):
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
_, body = client.get_object(self.url, self.token,
self.container, 'assembly',
resp_chunk_size=1024,
headers=headers)
self.compare_slo_to_chunks(body)
test_get_SLO_with_storlet.slow = 1
class TestSLOOnProxy(TestSLO): class TestSLOOnProxy(TestSLO):