Add support for claims

Implements blueprint claims-management
Change-Id: Icaa7c6df73e13200fc3ba03c34e90c5ac850eb46
This commit is contained in:
Angus Salkeld 2014-03-04 14:08:05 +10:00
parent 8e069942a5
commit a0b4feb488
9 changed files with 464 additions and 0 deletions

52
examples/claims.py Normal file
View File

@ -0,0 +1,52 @@
# Copyright (c) 2014 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from marconiclient.queues.v1 import client
URL = 'http://localhost:8888/v1'
cli = client.Client(URL)
queue = cli.queue('worker-jobs')
def send_jobs():
jobs = [
{'name': 'fluffy'},
{'name': 'scout'},
{'name': 'jo'}
]
queue.post([{'body': j,
'ttl': 360}
for j in jobs])
def process_jobs():
while True:
claim1 = queue.claim(ttl=500, grace=900, limit=2)
for msg in claim1:
claim_id = msg.claim_id
print('{claim_id} =? {id}'.format(claim_id=claim_id, id=claim1.id))
print('processing job %s' % (msg))
msg.delete()
claim2 = queue.claim(id=claim_id)
print(claim2)
print('nothing to do but have a sleep :)')
time.sleep(1)
if __name__ == '__main__':
send_jobs()
process_jobs()

View File

@ -162,6 +162,48 @@ class V1(api.Api):
}
},
'claim_create': {
'ref': 'queues/{queue_name}/claims',
'method': 'POST',
'required': ['queue_name'],
'properties': {
'queue_name': {'type': 'string'},
'limit': {'type': 'integer'},
'grace': {'type': 'integer'}
}
},
'claim_get': {
'ref': 'queues/{queue_name}/claims/{claim_id}',
'method': 'GET',
'required': ['queue_name', 'claim_id'],
'properties': {
'queue_name': {'type': 'string'},
'claim_id': {'type': 'string'}
}
},
'claim_update': {
'ref': 'queues/{queue_name}/claims/{claim_id}',
'method': 'PATCH',
'required': ['queue_name', 'claim_id'],
'properties': {
'queue_name': {'type': 'string'},
'claim_id': {'type': 'string'}
}
},
'claim_delete': {
'ref': 'queues/{queue_name}/claims/{claim_id}',
'method': 'DELETE',
'required': ['queue_name', 'claim_id'],
'properties': {
'queue_name': {'type': 'string'},
'claim_id': {'type': 'string'}
}
},
'health': {
'admin': True,
'ref': 'health',

View File

@ -0,0 +1,97 @@
# Copyright (c) 2014 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from marconiclient.queues.v1 import core
from marconiclient.queues.v1 import message
class Claim(object):
def __init__(self, queue, id=None,
ttl=None, grace=None, limit=None):
self._queue = queue
self.id = id
self._ttl = ttl
self._grace = grace
self._age = None
self._limit = limit
self._message_iter = None
if id is None:
self._create()
def __repr__(self):
return '<Claim id:{id} ttl:{ttl} age:{age}>'.format(id=self.id,
ttl=self.ttl,
age=self.age)
def _get(self):
req, trans = self._queue.client._request_and_transport()
claim_res = core.claim_get(trans, req, self._queue._name,
self.id)
self._age = claim_res['age']
self._ttl = claim_res['ttl']
self._grace = claim_res.get('grace')
msgs = claim_res.get('messages', [])
self._message_iter = message._MessageIterator(self._queue,
msgs)
def _create(self):
req, trans = self._queue.client._request_and_transport()
msgs = core.claim_create(trans, req,
self._queue._name,
ttl=self._ttl,
grace=self._grace,
limit=self._limit)
# extract the id from the first message
if msgs is not None:
self.id = msgs[0]['href'].split('=')[-1]
self._message_iter = message._MessageIterator(self._queue, msgs or [])
def __iter__(self):
if self._message_iter is None:
self._get()
return self._message_iter
@property
def age(self):
if self._age is None:
self._get()
return self._age
@property
def ttl(self):
if self._ttl is None:
self._get()
return self._ttl
def delete(self):
req, trans = self._queue.client._request_and_transport()
core.claim_delete(trans, req, self._queue._name, self.id)
def update(self, ttl=None, grace=None):
req, trans = self._queue.client._request_and_transport()
kwargs = {}
if ttl is not None:
kwargs['ttl'] = ttl
if grace is not None:
kwargs['grace'] = grace
res = core.claim_update(trans, req, self._queue._name, self.id,
**kwargs)
# if the update succeeds, update our attributes.
if ttl is not None:
self._ttl = ttl
if grace is not None:
self._grace = grace
return res

View File

@ -267,6 +267,73 @@ def message_delete(transport, request, queue_name, message_id, callback=None):
transport.send(request)
def claim_create(transport, request, queue_name, **kwargs):
"""Creates a Claim `claim_id` on the queue `queue_name`
:param transport: Transport instance to use
:type transport: `transport.base.Transport`
:param request: Request instance ready to be sent.
:type request: `transport.request.Request`
"""
request.operation = 'claim_create'
request.params['queue_name'] = queue_name
request.content = json.dumps(kwargs)
resp = transport.send(request)
return resp.deserialized_content
def claim_get(transport, request, queue_name, claim_id):
"""Gets a Claim `claim_id`
:param transport: Transport instance to use
:type transport: `transport.base.Transport`
:param request: Request instance ready to be sent.
:type request: `transport.request.Request`
"""
request.operation = 'claim_get'
request.params['queue_name'] = queue_name
request.params['claim_id'] = claim_id
resp = transport.send(request)
return resp.deserialized_content
def claim_update(transport, request, queue_name, claim_id, **kwargs):
"""Updates a Claim `claim_id`
:param transport: Transport instance to use
:type transport: `transport.base.Transport`
:param request: Request instance ready to be sent.
:type request: `transport.request.Request`
"""
request.operation = 'claim_update'
request.params['queue_name'] = queue_name
request.params['claim_id'] = claim_id
request.content = json.dumps(kwargs)
resp = transport.send(request)
return resp.deserialized_content
def claim_delete(transport, request, queue_name, claim_id):
"""Deletes a Claim `claim_id`
:param transport: Transport instance to use
:type transport: `transport.base.Transport`
:param request: Request instance ready to be sent.
:type request: `transport.request.Request`
"""
request.operation = 'claim_delete'
request.params['queue_name'] = queue_name
request.params['claim_id'] = claim_id
transport.send(request)
def shard_create(transport, request, shard_name, shard_data):
"""Creates a shard called `shard_name`

View File

@ -101,12 +101,23 @@ class Message(object):
# NOTE(flaper87): Is this really
# necessary? Should this be returned
# by Marconi?
# The url has two forms depending on if it has been claimed.
# /v1/queues/worker-jobs/messages/5c6939a8?claim_id=63c9a592
# or
# /v1/queues/worker-jobs/messages/5c6939a8
self._id = href.split('/')[-1]
if '?' in self._id:
self._id = self._id.split('?')[0]
def __repr__(self):
return '<Message id:{id} ttl:{ttl}>'.format(id=self._id,
ttl=self.ttl)
@property
def claim_id(self):
if '=' in self.href:
return self.href.split('=')[-1]
def delete(self):
req, trans = self.queue.client._request_and_transport()
core.message_delete(trans, req, self.queue._name, self._id)

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from marconiclient.queues.v1 import claim as claim_api
from marconiclient.queues.v1 import core
from marconiclient.queues.v1 import message
@ -208,3 +209,7 @@ class Queue(object):
**params)
return message._MessageIterator(self, msgs)
def claim(self, id=None, ttl=None, grace=None,
limit=None):
return claim_api.Claim(self, id=id, ttl=ttl, grace=grace, limit=limit)

View File

@ -0,0 +1,139 @@
# Copyright (c) 2014 Rackspace Hosting.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import mock
from marconiclient.queues.v1 import claim
from marconiclient.tests.queues import base
from marconiclient.transport import errors
from marconiclient.transport import response
class QueueV1ClaimUnitTest(base.QueuesTestBase):
def test_claim(self):
result = [{
'href': '/v1/queues/fizbit/messages/50b68a50d6f5b8c8a7c62b01',
'ttl': 800,
'age': 790,
'body': {'event': 'ActivateAccount', 'mode': 'active'}
}, {
'href': '/v1/queues/fizbit/messages/50b68a50d6f5b8c8a7c62b02',
'ttl': 800,
'age': 790,
'body': {'event': 'ActivateAccount', 'mode': 'active'}
}]
with mock.patch.object(self.transport, 'send',
autospec=True) as send_method:
resp = response.Response(None, json.dumps(result))
send_method.return_value = resp
claimed = self.queue.claim(ttl=60, grace=60)
# messages doesn't support len()
num_tested = 0
for num, msg in enumerate(claimed):
num_tested += 1
self.assertEqual(result[num]['href'], msg.href)
self.assertEqual(len(result), num_tested)
def test_claim_get_by_id(self):
result = {
'href': '/v1/queues/fizbit/messages/50b68a50d6cb01?claim_id=4524',
'age': 790,
'ttl': 800,
'messages': [{
'href': '/v1/queues/fizbit/messages/50b68a50d6f5b8c8a7c62b01',
'ttl': 800,
'age': 790,
'body': {'event': 'ActivateAccount', 'mode': 'active'}
}]}
with mock.patch.object(self.transport, 'send',
autospec=True) as send_method:
resp = response.Response(None, json.dumps(result))
send_method.return_value = resp
cl = self.queue.claim(id='5245432')
# messages doesn't support len()
num_tested = 0
for num, msg in enumerate(cl):
num_tested += 1
self.assertEqual(result['messages'][num]['href'], msg.href)
self.assertEqual(len(result['messages']), num_tested)
def test_claim_update(self):
with mock.patch.object(self.transport, 'send',
autospec=True) as send_method:
resp = response.Response(None, None)
send_method.return_value = resp
self.queue.claim(id='5245432').update(ttl=444, grace=987)
# NOTE(asalkeld): Nothing to assert here,
# just checking our way down to the transport
# doesn't crash.
def test_claim_delete(self):
with mock.patch.object(self.transport, 'send',
autospec=True) as send_method:
resp = response.Response(None, None)
send_method.return_value = resp
self.queue.claim(id='4225').delete()
# NOTE(asalkeld): Nothing to assert here,
# just checking our way down to the transport
# doesn't crash.
class QueuesV1ClaimFunctionalTest(base.QueuesTestBase):
def test_message_claim_functional(self):
queue = self.client.queue("test_queue")
queue._get_transport = mock.Mock(return_value=self.transport)
messages = [{'ttl': 60, 'body': 'Post It 1!'}]
queue.post(messages)
messages = queue.claim(ttl=120, grace=120)
self.assertTrue(isinstance(messages, claim.Claim))
self.assertGreaterEqual(len(list(messages)), 0)
def test_claim_get_functional(self):
queue = self.client.queue("test_queue")
queue._get_transport = mock.Mock(return_value=self.transport)
res = queue.claim(ttl=100, grace=100)
claim_id = res.id
cl = queue.claim(id=claim_id)
self.assertEqual(cl.id, claim_id)
def test_claim_create_delete_functional(self):
queue = self.client.queue("test_queue")
queue._get_transport = mock.Mock(return_value=self.transport)
messages = [{'ttl': 60, 'body': 'Post It 1!'}]
queue.post(messages)
cl = queue.claim(ttl=120, grace=120)
claim_id = cl.id
cl.delete()
self.assertRaises(errors.ResourceNotFound, queue.claim, id=claim_id)

View File

@ -0,0 +1,26 @@
# Copyright (c) 2014 Rackspace Hosting.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from marconiclient.tests.queues import claims
from marconiclient.transport import http
class QueuesV1ClaimHttpFunctionalTest(claims.QueuesV1ClaimFunctionalTest):
is_functional = True
transport_cls = http.HttpTransport
url = 'http://127.0.0.1:8888/v1'
version = 1

View File

@ -0,0 +1,25 @@
# Copyright (c) Rackspace Hosting.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from marconiclient.tests.queues import claims
from marconiclient.transport import http
class QueuesV1ClaimsHttpUnitTest(claims.QueueV1ClaimUnitTest):
transport_cls = http.HttpTransport
url = 'http://127.0.0.1:8888/v1'
version = 1