« back to all changes in this revision
Viewing changes to grackle/tests/test_client.py
- browse files at revision 34
- compare with another revision
- download diff
- download tarball
- view history from revision 34
- Committer: Aaron Bentley
- Date: 2012-01-16 20:31:23 UTC
- Revision ID: aaron@canonical.com-20120116203123-bxg29ktwtq19al75
Cleanup
- files removed:
- grackle/error.py
- files modified:
- grackle-put-message
added
removed
grackle/tests/test_client.py
2
2
HTTPServer,
3
3
BaseHTTPRequestHandler,
4
4
)
5
from email.message import Message
6
from email.mime.multipart import MIMEMultipart
7
from email.mime.text import MIMEText
8
5
import httplib
9
6
import logging
10
7
import os
14
11
import sys
15
12
from unittest import TestCase
16
13
from urlparse import urlparse
14
from urlparse import parse_qs
17
15
18
16
from testtools import ExpectedException
19
17
20
18
from grackle.client import (
21
19
GrackleClient,
22
UnparsableDateRange,
23
UnsupportedDisplayType,
24
20
UnsupportedOrder,
25
21
)
26
from grackle.store import (
27
make_json_message,
28
MemoryStore,
29
)
30
31
32
def make_message(message_id, body='body', headers=None, hidden=False):
33
if headers is None:
34
headers = {}
35
headers['Message-Id'] = message_id
36
message = {
37
'message_id': message_id,
38
'headers': headers,
39
'thread_id': message_id,
40
'date': headers.get('date', '2005-01-01'),
41
'subject': headers.get('subject', 'subject'),
42
'author': headers.get('author', 'author'),
43
'hidden': hidden,
44
'attachments': [],
45
'replies': headers.get('in-reply-to', None),
46
'body': body,
47
}
48
return message
49
50
51
def make_mime_message(message_id, body='body', headers=None, hidden=False,
52
attachment_type=None):
53
if headers is None:
54
headers = {}
55
parts = MIMEMultipart()
56
parts.attach(MIMEText(body))
57
if attachment_type is not None:
58
attachment = Message()
59
attachment.set_payload('attactment data.')
60
attachment['Content-Type'] = attachment_type
61
attachment['Content-Disposition'] = 'attachment; filename="file.ext"'
62
parts.attach(attachment)
63
message = Message()
64
message.set_payload(parts.as_string())
65
for key, value in headers.items():
66
message[key] = value
67
return make_json_message(message_id, message.as_string())
68
69
70
class ForkedFakeService:
22
23
24
def threaded_messages(messages):
25
threads = {}
26
count = 0
27
pending = []
28
for message in messages:
29
if message.get('in_reply_to') is None:
30
threads[message['message_id']] = [message]
31
count += 1
32
else:
33
pending.append(message)
34
for message in pending:
35
threads[message['in_reply_to']].append(message)
36
return threads.values()
37
38
39
class GrackleStore:
40
"""A memory-backed message store."""
41
42
def __init__(self, messages):
43
"""Constructor."""
44
self.messages = messages
45
46
def get_messages(self, archive_id, query_string):
47
"""Return matching messages.
48
49
:param archive_id: The archive to retrieve from.
50
:param query_string: Contains 'parameters', which is a JSON-format
51
string describing parameters.
52
"""
53
query = parse_qs(query_string)
54
parameters = simplejson.loads(query['parameters'][0])
55
order = parameters.get('order')
56
messages = self.messages[archive_id]
57
if order is not None :
58
if order not in SUPPORTED_ORDERS:
59
raise UnsupportedOrder
60
elif order.startswith('thread_'):
61
threaded = threaded_messages(messages)
62
messages = []
63
if order == 'thread_subject':
64
threaded.sort(key=lambda t: t[0]['subject'])
65
if order == 'thread_oldest':
66
threaded.sort(key=lambda t: min(m['date'] for m in t))
67
if order == 'thread_newest':
68
threaded.sort(key=lambda t: max(m['date'] for m in t))
69
for thread in threaded:
70
messages.extend(thread)
71
else:
72
messages.sort(key=lambda m: m[order])
73
new_messages = []
74
for message in messages:
75
if (
76
not parameters['include_hidden']
77
and message.get('hidden', False)):
78
continue
79
80
if ('message_ids' in parameters and
81
message['message_id'] not in parameters['message_ids']):
82
continue
83
message = dict(message)
84
if 'headers' in parameters:
85
headers = dict(
86
(k, v) for k, v in message['headers'].iteritems()
87
if k in parameters['headers'])
88
message['headers'] = headers
89
max_body = parameters.get('max_body_length')
90
if max_body is not None:
91
message['body'] = message['body'][:max_body]
92
new_messages.append(message)
93
messages = new_messages
94
limit = parameters.get('limit', 100)
95
memo = parameters.get('memo')
96
message_id_indices = dict(
97
(m['message_id'], idx) for idx, m in enumerate(messages))
98
if memo is None:
99
start = 0
100
else:
101
start = message_id_indices[memo.encode('rot13')]
102
if start > 0:
103
previous_memo = messages[start - 1]['message_id'].encode('rot13')
104
else:
105
previous_memo = None
106
end = min(start + limit, len(messages))
107
if end < len(messages):
108
next_memo = messages[end]['message_id'].encode('rot13')
109
else:
110
next_memo = None
111
messages = messages[start:end]
112
113
response = {
114
'messages': messages,
115
'next_memo': next_memo,
116
'previous_memo': previous_memo
117
}
118
return response
119
120
121
122
class ForkedFake:
71
123
"""A Grackle service fake, as a ContextManager."""
72
124
73
def __init__(self, port, message_archives=None, write_logs=False):
125
def __init__(self, port, messages=None, write_logs=False):
74
126
"""Constructor.
75
76
:param port: The tcp port to use.
77
:param message_archives: A dict of lists of dicts representing
78
archives of messages. The outer dict represents the archive,
79
the list represents the list of messages for that archive.
127
:param port: The tcp port to use
128
:param messages: A dict of lists of dicts representing messages. The
129
outer dict represents the archive, the list represents the list of
130
messages for that archive.
80
131
:param write_logs: If true, log messages will be written to stdout.
81
132
"""
82
133
self.pid = None
83
134
self.port = port
84
if message_archives is None:
85
self.message_archives = {}
135
if messages is None:
136
self.messages = {}
86
137
else:
87
self.message_archives = message_archives
138
self.messages = messages
88
139
self.read_end, self.write_end = os.pipe()
89
140
self.write_logs = write_logs
90
141
91
142
@staticmethod
92
def from_client(client, message_archives=None):
93
"""Instantiate a ForkedFakeService from the client.
143
def from_client(client, messages=None):
144
"""Instantiate a ForkedFake from the client.
94
145
95
:param port: The client to provide service for.
96
:param message_archives: A dict of lists of dicts representing
97
archives of messages. The outer dict represents the archive,
98
the list represents the list of messages for that archive.
146
:param port: The client to provide service for.
147
:param messages: A dict of lists of dicts representing messages. The
148
outer dict represents the archive, the list represents the list of
149
messages for that archive.
99
150
"""
100
return ForkedFakeService(client.port, message_archives)
151
return ForkedFake(client.port, messages)
101
152
102
153
def is_ready(self):
103
154
"""Tell the parent process that the server is ready for writes."""
118
169
def start_server(self):
119
170
"""Start the HTTP server."""
120
171
service = HTTPServer(('', self.port), FakeGrackleRequestHandler)
121
service.store = MemoryStore(self.message_archives)
122
for archive_id, messages in service.store.message_archives.iteritems():
172
service.store = GrackleStore(self.messages)
173
for archive_id, messages in service.store.messages.iteritems():
123
174
for message in messages:
124
175
message.setdefault('headers', {})
125
176
self.is_ready()
132
183
os.kill(self.pid, SIGKILL)
133
184
134
185
186
SUPPORTED_ORDERS = set(
187
['date', 'author', 'subject', 'thread_newest', 'thread_oldest',
188
'thread_subject'])
189
190
135
191
class FakeGrackleRequestHandler(BaseHTTPRequestHandler):
136
192
"""A request handler that forwards to server.store."""
137
193
143
199
def do_POST(self):
144
200
"""Create a message on POST."""
145
201
message = self.rfile.read(int(self.headers['content-length']))
146
scheme, netloc, path, params, query_string, fragments = (
147
urlparse(self.path))
148
parts = path.split('/')
149
if parts[1] == 'archive' and len(parts) == 4:
150
try:
151
# This expected path is /archive/archive_id/message_id.
152
self.server.store.put_message(parts[2], parts[3], message)
153
self.send_response(httplib.CREATED)
154
self.end_headers()
155
self.wfile.close()
156
except:
157
self.send_error(httplib.BAD_REQUEST)
202
if message == 'This is a message':
203
self.send_response(httplib.CREATED)
204
self.end_headers()
205
self.wfile.close()
206
else:
207
self.send_error(httplib.BAD_REQUEST)
158
208
159
209
def do_GET(self):
160
210
"""Retrieve a list of messages on GET."""
168
218
self.send_response(httplib.OK)
169
219
self.end_headers()
170
220
self.wfile.write(simplejson.dumps(response))
171
except Exception, error:
172
self.send_response(
173
httplib.BAD_REQUEST, error.__doc__)
221
except UnsupportedOrder:
222
self.send_response(httplib.BAD_REQUEST)
223
self.wfile.write('Unsupported order')
174
224
return
175
225
176
226
def log_message(self, format, *args):
177
227
"""Override log_message to use standard Python logging."""
178
228
message = "%s - - [%s] %s\n" % (
179
self.address_string(), self.log_date_time_string(), format % args)
229
self.address_string(), self.log_date_time_string(), format%args)
180
230
self.logger.info(message)
181
231
182
232
183
233
class TestPutMessage(TestCase):
184
234
185
235
def test_put_message(self):
186
client = GrackleClient('localhost', 8420)
187
message_archives = {'arch1': []}
188
with ForkedFakeService.from_client(client, message_archives):
189
client.put_message('arch1', 'id1', StringIO('This is a message'))
190
response = client.get_messages('arch1')
191
self.assertEqual(1, len(response['messages']))
192
message = response['messages'][0]
193
self.assertEqual('id1', message['message_id'])
194
195
def test_put_message_without_archive(self):
196
client = GrackleClient('localhost', 8421)
197
message_archives = {'arch1': []}
198
with ForkedFakeService.from_client(client, message_archives):
236
client = GrackleClient('localhost', 8436)
237
with ForkedFake.from_client(client):
238
client.put_message('arch1', 'asdf', StringIO('This is a message'))
199
239
with ExpectedException(Exception, 'wtf'):
200
client.put_message('no-archive', 'id1', StringIO('message'))
240
client.put_message('arch1', 'asdf',
241
StringIO('This is not a message'))
201
242
202
243
203
244
class TestGetMessages(TestCase):
207
248
208
249
def assertMessageIDs(self, ids, messages):
209
250
self.assertIDOrder(
210
sorted(ids), sorted(messages, key=lambda m: m['message_id']))
251
sorted(ids), sorted(messages, key=lambda m:m['message_id']))
211
252
212
253
def test_get_messages(self):
213
client = GrackleClient('localhost', 8430)
214
archive = {
215
'baz': [make_message('foo'), make_message('bar')]}
216
with ForkedFakeService.from_client(client, archive):
254
client = GrackleClient('localhost', 8435)
255
with ForkedFake.from_client(client,
256
{'baz':
257
[{'message_id': 'foo'},
258
{'message_id': 'bar'}]}):
217
259
response = client.get_messages('baz')
218
260
self.assertEqual(['bar', 'foo'], sorted(m['message_id'] for m in
219
261
response['messages']))
222
264
223
265
def test_get_messages_by_id(self):
224
266
client = GrackleClient('localhost', 8437)
225
archive = {
226
'baz': [make_message('foo'), make_message('bar')]}
227
with ForkedFakeService.from_client(client, archive):
267
with ForkedFake.from_client(client,
268
{'baz':
269
[{'message_id': 'foo'},
270
{'message_id': 'bar'}]}):
228
271
response = client.get_messages('baz', message_ids=['foo'])
229
272
message, = response['messages']
230
273
self.assertEqual('foo', message['message_id'])
231
274
232
275
def test_get_messages_batching(self):
233
276
client = GrackleClient('localhost', 8438)
234
archive = {'baz': [make_message('foo'), make_message('bar')]}
235
with ForkedFakeService.from_client(client, archive):
277
with ForkedFake.from_client(client,
278
{'baz':
279
[{'message_id': 'foo'},
280
{'message_id': 'bar'}]}):
236
281
response = client.get_messages('baz', limit=1)
237
282
self.assertEqual(1, len(response['messages']))
238
283
messages = response['messages']
244
289
245
290
def get_messages_member_order_test(self, key):
246
291
client = GrackleClient('localhost', 8439)
247
archive = {
248
'baz': [
249
make_message('foo', headers={key: '2011-03-25'}),
250
make_message('bar', headers={key: '2011-03-24'}),
251
]}
252
with ForkedFakeService.from_client(client, archive):
292
with ForkedFake.from_client(client,
293
{'baz': [{'message_id': 'foo', key: '2011-03-25'},
294
{'message_id': 'bar', key: '2011-03-24'}]}):
253
295
response = client.get_messages('baz')
254
296
self.assertIDOrder(['foo', 'bar'], response['messages'])
255
297
response = client.get_messages('baz', order=key)
265
307
self.get_messages_member_order_test('subject')
266
308
267
309
def test_get_messages_thread_subject_order(self):
268
archive = {
269
'baz': [
270
make_message('bar', headers={'subject': 'y'}),
271
make_message('qux', headers={'subject': 'z'}),
272
make_message('foo', headers={'subject': 'x',
273
'in-reply-to': 'qux'}),
274
]}
275
310
client = GrackleClient('localhost', 8439)
276
with ForkedFakeService.from_client(client, archive):
311
with ForkedFake.from_client(client, {'baz': [
312
{'message_id': 'bar', 'subject': 'y'},
313
{'message_id': 'qux', 'subject': 'z'},
314
{'message_id': 'foo', 'subject': 'x', 'in_reply_to': 'qux'},
315
]}):
277
316
response = client.get_messages('baz')
278
317
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
279
318
response = client.get_messages('baz', order='subject')
283
322
284
323
def test_get_messages_thread_oldest_order(self):
285
324
client = GrackleClient('localhost', 8439)
286
archive = {
287
'baz': [
288
make_message('bar', headers={'date': 'x'}),
289
make_message('qux', headers={'date': 'z'}),
290
make_message('foo', headers={'date': 'y',
291
'in-reply-to': 'qux'}),
292
]}
293
with ForkedFakeService.from_client(client, archive):
325
with ForkedFake.from_client(client, {'baz': [
326
{'message_id': 'bar', 'date': 'x'},
327
{'message_id': 'qux', 'date': 'z'},
328
{'message_id': 'foo', 'date': 'y', 'in_reply_to': 'qux'},
329
]}):
294
330
response = client.get_messages('baz')
295
331
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
296
332
response = client.get_messages('baz', order='date')
300
336
301
337
def test_get_messages_thread_newest_order(self):
302
338
client = GrackleClient('localhost', 8439)
303
archive = {
304
'baz': [
305
make_message('bar', headers={'date': 'x'}),
306
make_message('qux', headers={'date': 'w'}),
307
make_message('foo', headers={'date': 'y',
308
'in-reply-to': 'bar'}),
309
make_message('baz', headers={'date': 'z',
310
'in-reply-to': 'qux'}),
311
]}
312
with ForkedFakeService.from_client(client, archive):
339
with ForkedFake.from_client(client, {'baz': [
340
{'message_id': 'bar', 'date': 'x'},
341
{'message_id': 'qux', 'date': 'w'},
342
{'message_id': 'foo', 'date': 'y', 'in_reply_to': 'bar'},
343
{'message_id': 'baz', 'date': 'z', 'in_reply_to': 'qux'},
344
]}):
313
345
response = client.get_messages('baz', order='date')
314
346
self.assertIDOrder(
315
347
['qux', 'bar', 'foo', 'baz'], response['messages'])
319
351
320
352
def test_get_messages_unsupported_order(self):
321
353
client = GrackleClient('localhost', 8439)
322
archive = {
323
'baz': [
324
make_message('foo', headers={'date': '2011-03-25'}),
325
make_message('foo', headers={'date': '2011-03-24'}),
326
]}
327
with ForkedFakeService.from_client(client, archive):
328
with ExpectedException(UnsupportedOrder, ''):
354
with ForkedFake.from_client(client,
355
{'baz': [{'message_id': 'foo', 'date': '2011-03-25'},
356
{'message_id': 'bar', 'date': '2011-03-24'}]}):
357
with ExpectedException(UnsupportedOrder):
329
358
client.get_messages('baz', order='nonsense')
330
359
331
360
def test_get_messages_headers_no_headers(self):
332
361
client = GrackleClient('localhost', 8440)
333
archive = {'baz': [make_message('foo')]}
334
with ForkedFakeService.from_client(client, archive):
362
with ForkedFake.from_client(client,
363
{'baz': [
364
{'message_id': 'foo'}
365
]}):
335
366
response = client.get_messages('baz', headers=[
336
367
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
337
368
first_message = response['messages'][0]
340
371
341
372
def test_get_messages_headers_exclude_headers(self):
342
373
client = GrackleClient('localhost', 8441)
343
archive = {
344
'baz': [make_message('foo', headers={'From': 'me'})]}
345
with ForkedFakeService.from_client(client, archive):
374
with ForkedFake.from_client(client,
375
{'baz': [
376
{'message_id': 'foo', 'headers': {'From': 'me'}}
377
]}):
346
378
response = client.get_messages('baz', headers=[
347
379
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
348
380
first_message = response['messages'][0]
351
383
352
384
def test_get_messages_headers_include_headers(self):
353
385
client = GrackleClient('localhost', 8442)
354
archive = {
355
'baz': [
356
make_message('foo', headers={'From': 'me', 'To': 'you'})]}
357
with ForkedFakeService.from_client(client, archive):
386
with ForkedFake.from_client(client,
387
{'baz': [
388
{'message_id': 'foo', 'headers': {'From': 'me', 'To': 'you'}}
389
]}):
358
390
response = client.get_messages('baz', headers=[
359
391
'From', 'To'])
360
392
first_message = response['messages'][0]
363
395
364
396
def test_get_messages_max_body_length(self):
365
397
client = GrackleClient('localhost', 8443)
366
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
367
with ForkedFakeService.from_client(client, archive):
398
with ForkedFake.from_client(client,
399
{'baz': [
400
{'message_id': 'foo', 'body': u'abcdefghi'}
401
]}):
368
402
response = client.get_messages('baz', max_body_length=3)
369
403
first_message = response['messages'][0]
370
404
self.assertEqual('abc', first_message['body'])
371
405
372
406
def test_include_hidden(self):
373
407
client = GrackleClient('localhost', 8444)
374
archive = {
375
'baz': [
376
make_message('foo', hidden=True),
377
make_message('bar', hidden=False),
378
]}
379
with ForkedFakeService.from_client(client, archive):
408
with ForkedFake.from_client(client,
409
{'baz': [
410
{'message_id': 'foo', 'hidden': True},
411
{'message_id': 'bar', 'hidden': False}
412
]}):
380
413
response = client.get_messages('baz', include_hidden=True)
381
414
self.assertMessageIDs(['bar', 'foo'], response['messages'])
382
415
response = client.get_messages('baz', include_hidden=False)
383
416
self.assertMessageIDs(['bar'], response['messages'])
384
417
385
def test_display_type_unknown_value(self):
386
client = GrackleClient('localhost', 8445)
387
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
388
with ForkedFakeService.from_client(client, archive):
389
with ExpectedException(UnsupportedDisplayType, ''):
390
client.get_messages('baz', display_type='unknown')
391
392
def test_display_type_headers_only(self):
393
client = GrackleClient('localhost', 8446)
394
archive = {
395
'baz': [
396
make_message('foo', body=u'abcdefghi',
397
headers={'From': 'me', 'To': 'you'})]}
398
with ForkedFakeService.from_client(client, archive):
399
response = client.get_messages('baz', display_type='headers-only')
400
first_message = response['messages'][0]
401
self.assertEqual('foo', first_message['message_id'])
402
self.assertEqual(
403
{'From': 'me', 'Message-Id': 'foo', 'To': 'you'},
404
first_message['headers'])
405
self.assertNotIn('body', first_message)
406
407
def test_display_type_text_only(self):
408
client = GrackleClient('localhost', 8446)
409
archive = {
410
'baz': [
411
make_mime_message(
412
'foo', 'abcdefghi',
413
headers={'From': 'me', 'To': 'you'},
414
attachment_type='text/x-diff')]}
415
with ForkedFakeService.from_client(client, archive):
416
response = client.get_messages('baz', display_type='text-only')
417
first_message = response['messages'][0]
418
self.assertEqual('foo', first_message['message_id'])
419
self.assertEqual('me', first_message['headers']['From'])
420
self.assertEqual('you', first_message['headers']['To'])
421
self.assertEqual(archive['baz'][0]['body'], first_message['body'])
422
423
def test_display_type_all(self):
424
client = GrackleClient('localhost', 8447)
425
archive = {
426
'baz': [
427
make_mime_message(
428
'foo', 'abcdefghi',
429
headers={'From': 'me', 'To': 'you'},
430
attachment_type='text/x-diff')]}
431
with ForkedFakeService.from_client(client, archive):
432
response = client.get_messages('baz', display_type='all')
433
first_message = response['messages'][0]
434
self.assertEqual('foo', first_message['message_id'])
435
self.assertEqual('me', first_message['headers']['From'])
436
self.assertEqual('you', first_message['headers']['To'])
437
self.assertEqual(archive['baz'][0]['body'], first_message['body'])
438
439
def test_date_range(self):
440
client = GrackleClient('localhost', 8448)
441
archive = {
442
'baz': [
443
make_mime_message(
444
'foo', 'abcdefghi', headers={'date': '2011-12-31'}),
445
make_mime_message(
446
'bar', 'abcdefghi', headers={'date': '2012-01-01'}),
447
make_mime_message(
448
'qux', 'abcdefghi', headers={'date': '2012-01-15'}),
449
make_mime_message(
450
'naf', 'abcdefghi', headers={'date': '2012-01-31'}),
451
make_mime_message(
452
'doh', 'abcdefghi', headers={'date': '2012-02-01'}),
453
]}
454
with ForkedFakeService.from_client(client, archive):
455
response = client.get_messages(
456
'baz', date_range='2012-01-01..2012-01-31')
457
ids = sorted(m['message_id'] for m in response['messages'])
458
self.assertEqual(['bar', 'naf', 'qux'], ids)
459
460
def test_date_range_unparsabledaterange(self):
461
client = GrackleClient('localhost', 8449)
462
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
463
with ForkedFakeService.from_client(client, archive):
464
with ExpectedException(UnparsableDateRange, ''):
465
client.get_messages('baz', date_range='2012-01-01')
466
467
def test_date_range_unparsabledaterange_missing_part(self):
468
client = GrackleClient('localhost', 8450)
469
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
470
with ForkedFakeService.from_client(client, archive):
471
with ExpectedException(UnparsableDateRange, ''):
472
client.get_messages('baz', date_range='2012-01-01..')
473
474
def test_date_range_unparsabledaterange_extra_part(self):
475
client = GrackleClient('localhost', 8451)
476
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
477
with ForkedFakeService.from_client(client, archive):
478
with ExpectedException(UnparsableDateRange, ''):
479
client.get_messages('baz', date_range='2012-01..12-02..12-03')
Loggerhead is a web-based interface for Breezy
Version: 2.0.1