« back to all changes in this revision
Viewing changes to grackle/tests/test_client.py
- browse files at revision 40
- compare with another revision
- download diff
- download tarball
- view history from revision 40
- Committer: Curtis Hovey
- Date: 2012-02-14 22:52:55 UTC
- Revision ID: curtis.hovey@canonical.com-20120214225255-7cwc33qlpmaztebo
Dates must be a value.
- files modified:
- grackle-put-message
added
removed
grackle/tests/test_client.py
2
2
HTTPServer,
3
3
BaseHTTPRequestHandler,
4
4
)
5
from email.message import Message
6
from email.mime.multipart import MIMEMultipart
7
from email.mime.text import MIMEText
5
8
import httplib
9
import logging
6
10
import os
7
11
from signal import SIGKILL
8
12
import simplejson
9
13
from StringIO import StringIO
14
import sys
10
15
from unittest import TestCase
11
16
from urlparse import urlparse
17
from urlparse import parse_qs
12
18
13
19
from testtools import ExpectedException
14
20
15
21
from grackle.client import (
16
22
GrackleClient,
23
UnparsableDateRange,
24
UnsupportedDisplayType,
25
UnsupportedOrder,
17
26
)
18
27
19
28
20
class ForkedFake:
21
22
def __init__(self, port, messages=None):
29
def make_message(message_id, body='body', headers=None, hidden=False):
30
if headers is None:
31
headers = {}
32
headers['Message-Id'] = message_id
33
message = {
34
'message_id': message_id,
35
'headers': headers,
36
'thread_id': message_id,
37
'date': headers.get('date', '2005-01-01'),
38
'subject': headers.get('subject', 'subject'),
39
'author': headers.get('author', 'author'),
40
'hidden': hidden,
41
'attachments': [],
42
'replies': headers.get('in-reply-to', None),
43
'body': body,
44
}
45
return message
46
47
48
def make_mime_message(message_id, body='body', headers=None, hidden=False,
49
attachment_type=None):
50
message = MIMEMultipart()
51
message.attach(MIMEText(body))
52
if attachment_type is not None:
53
attachment = Message()
54
attachment.set_payload('attactment data.')
55
attachment['Content-Type'] = attachment_type
56
attachment['Content-Disposition'] = 'attachment; filename="file.ext"'
57
message.attach(attachment)
58
return make_message(message_id, message.get_payload(), headers, hidden)
59
60
61
def threaded_messages(messages):
62
threads = {}
63
count = 0
64
pending = []
65
for message in messages:
66
if message.get('replies') is None:
67
threads[message['message_id']] = [message]
68
count += 1
69
else:
70
pending.append(message)
71
for message in pending:
72
threads[message['replies']].append(message)
73
return threads.values()
74
75
76
class GrackleStore:
77
"""A memory-backed message store."""
78
79
def __init__(self, messages):
80
"""Constructor."""
81
self.messages = messages
82
83
@staticmethod
84
def is_multipart(message):
85
return isinstance(message['body'], list)
86
87
def get_messages(self, archive_id, query_string):
88
"""Return matching messages.
89
90
:param archive_id: The archive to retrieve from.
91
:param query_string: Contains 'parameters', which is a JSON-format
92
string describing parameters.
93
"""
94
query = parse_qs(query_string)
95
parameters = simplejson.loads(query['parameters'][0])
96
order = parameters.get('order')
97
messages = self.messages[archive_id]
98
if order is not None:
99
if order not in SUPPORTED_ORDERS:
100
raise UnsupportedOrder
101
elif order.startswith('thread_'):
102
threaded = threaded_messages(messages)
103
messages = []
104
if order == 'thread_subject':
105
threaded.sort(key=lambda t: t[0]['subject'])
106
if order == 'thread_oldest':
107
threaded.sort(key=lambda t: min(m['date'] for m in t))
108
if order == 'thread_newest':
109
threaded.sort(key=lambda t: max(m['date'] for m in t))
110
for thread in threaded:
111
messages.extend(thread)
112
else:
113
messages.sort(key=lambda m: m[order])
114
display_type = parameters.get('display_type', 'all')
115
if display_type not in SUPPORTED_DISPLAY_TYPES:
116
raise UnsupportedDisplayType
117
if 'date_range' in parameters:
118
try:
119
start_date, end_date = parameters['date_range'].split('..')
120
if not start_date or not end_date:
121
raise UnparsableDateRange
122
except ValueError:
123
raise UnparsableDateRange
124
new_messages = []
125
for message in messages:
126
if (not parameters['include_hidden'] and message['hidden']):
127
continue
128
if ('message_ids' in parameters
129
and message['message_id'] not in parameters['message_ids']):
130
continue
131
if ('date_range' in parameters
132
and (message['date'] < start_date
133
or message['date'] > end_date)):
134
continue
135
message = dict(message)
136
if 'headers' in parameters:
137
headers = dict(
138
(k, v) for k, v in message['headers'].iteritems()
139
if k in parameters['headers'])
140
message['headers'] = headers
141
if display_type == 'headers-only':
142
del message['body']
143
elif display_type == 'text-only' and self.is_multipart(message):
144
text_parts = [
145
part.get_payload() for part in message['body']
146
if part.get_content_type() == 'text/plain']
147
message['body'] = '\n\n'.join(text_parts)
148
elif display_type == 'all' and self.is_multipart(message):
149
parts = [str(part.get_payload()) for part in message['body']]
150
message['body'] = '\n\n'.join(parts)
151
max_body = parameters.get('max_body_length')
152
if max_body is not None and display_type != 'headers-only':
153
message['body'] = message['body'][:max_body]
154
new_messages.append(message)
155
messages = new_messages
156
limit = parameters.get('limit', 100)
157
memo = parameters.get('memo')
158
message_id_indices = dict(
159
(m['message_id'], idx) for idx, m in enumerate(messages))
160
if memo is None:
161
start = 0
162
else:
163
start = message_id_indices[memo.encode('rot13')]
164
if start > 0:
165
previous_memo = messages[start - 1]['message_id'].encode('rot13')
166
else:
167
previous_memo = None
168
end = min(start + limit, len(messages))
169
if end < len(messages):
170
next_memo = messages[end]['message_id'].encode('rot13')
171
else:
172
next_memo = None
173
messages = messages[start:end]
174
175
response = {
176
'messages': messages,
177
'next_memo': next_memo,
178
'previous_memo': previous_memo
179
}
180
return response
181
182
183
class ForkedFakeService:
184
"""A Grackle service fake, as a ContextManager."""
185
186
def __init__(self, port, messages=None, write_logs=False):
187
"""Constructor.
188
189
:param port: The tcp port to use.
190
:param messages: A dict of lists of dicts representing messages. The
191
outer dict represents the archive, the list represents the list of
192
messages for that archive.
193
:param write_logs: If true, log messages will be written to stdout.
194
"""
23
195
self.pid = None
24
196
self.port = port
25
self.messages = messages
197
if messages is None:
198
self.messages = {}
199
else:
200
self.messages = messages
26
201
self.read_end, self.write_end = os.pipe()
202
self.write_logs = write_logs
203
204
@staticmethod
205
def from_client(client, messages=None):
206
"""Instantiate a ForkedFakeService from the client.
207
208
:param port: The client to provide service for.
209
:param messages: A dict of lists of dicts representing messages. The
210
outer dict represents the archive, the list represents the list of
211
messages for that archive.
212
"""
213
return ForkedFakeService(client.port, messages)
27
214
28
215
def is_ready(self):
216
"""Tell the parent process that the server is ready for writes."""
29
217
os.write(self.write_end, 'asdf')
30
218
31
219
def __enter__(self):
220
"""Run the service.
221
222
Fork and start a server in the child. Return when the server is ready
223
for use."""
32
224
pid = os.fork()
33
225
if pid == 0:
34
226
self.start_server()
37
229
return
38
230
39
231
def start_server(self):
232
"""Start the HTTP server."""
40
233
service = HTTPServer(('', self.port), FakeGrackleRequestHandler)
41
service.messages = self.messages
234
service.store = GrackleStore(self.messages)
235
for archive_id, messages in service.store.messages.iteritems():
236
for message in messages:
237
message.setdefault('headers', {})
42
238
self.is_ready()
239
if self.write_logs:
240
logging.basicConfig(
241
stream=sys.stderr, level=logging.INFO)
43
242
service.serve_forever()
44
243
45
244
def __exit__(self, exc_type, exc_val, traceback):
46
245
os.kill(self.pid, SIGKILL)
47
246
48
247
248
SUPPORTED_DISPLAY_TYPES = set(['all', 'text-only', 'headers-only'])
249
250
251
SUPPORTED_ORDERS = set(
252
['date', 'author', 'subject', 'thread_newest', 'thread_oldest',
253
'thread_subject'])
254
255
49
256
class FakeGrackleRequestHandler(BaseHTTPRequestHandler):
257
"""A request handler that forwards to server.store."""
258
259
def __init__(self, *args, **kwargs):
260
"""Constructor. Sets up logging."""
261
self.logger = logging.getLogger('http')
262
BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
50
263
51
264
def do_POST(self):
265
"""Create a message on POST."""
52
266
message = self.rfile.read(int(self.headers['content-length']))
53
267
if message == 'This is a message':
54
268
self.send_response(httplib.CREATED)
58
272
self.send_error(httplib.BAD_REQUEST)
59
273
60
274
def do_GET(self):
61
scheme, netloc, path, params, query, fragments = urlparse(self.path)
62
archive = os.path.split(path)[1]
63
self.send_response(httplib.OK)
64
self.end_headers()
65
self.wfile.write(simplejson.dumps(self.server.messages[archive]))
66
67
68
def fake_grackle_service(client, messages=None):
69
if messages is None:
70
messages = {}
71
return ForkedFake(client.port, messages)
275
"""Retrieve a list of messages on GET."""
276
scheme, netloc, path, params, query_string, fragments = (
277
urlparse(self.path))
278
parts = path.split('/')
279
if parts[1] == 'archive':
280
try:
281
response = self.server.store.get_messages(
282
parts[2], query_string)
283
self.send_response(httplib.OK)
284
self.end_headers()
285
self.wfile.write(simplejson.dumps(response))
286
except Exception, error:
287
self.send_response(
288
httplib.BAD_REQUEST, error.__doc__)
289
return
290
291
def log_message(self, format, *args):
292
"""Override log_message to use standard Python logging."""
293
message = "%s - - [%s] %s\n" % (
294
self.address_string(), self.log_date_time_string(), format % args)
295
self.logger.info(message)
72
296
73
297
74
298
class TestPutMessage(TestCase):
75
299
76
300
def test_put_message(self):
77
301
client = GrackleClient('localhost', 8436)
78
with fake_grackle_service(client):
302
with ForkedFakeService.from_client(client):
79
303
client.put_message('arch1', 'asdf', StringIO('This is a message'))
80
304
with ExpectedException(Exception, 'wtf'):
81
305
client.put_message('arch1', 'asdf',
84
308
85
309
class TestGetMessages(TestCase):
86
310
311
def assertIDOrder(self, ids, messages):
312
self.assertEqual(ids, [m['message_id'] for m in messages])
313
314
def assertMessageIDs(self, ids, messages):
315
self.assertIDOrder(
316
sorted(ids), sorted(messages, key=lambda m: m['message_id']))
317
87
318
def test_get_messages(self):
88
319
client = GrackleClient('localhost', 8435)
89
with fake_grackle_service(client,
90
{'baz':
91
[{'message-id': 'foo'},
92
{'message-id': 'bar'}]}):
93
response = client.get_messages('baz', message_ids=['a'])
94
self.assertEqual(['bar', 'foo'], sorted(m['message-id'] for m in
95
response))
320
archive = {
321
'baz': [make_message('foo'), make_message('bar')]}
322
with ForkedFakeService.from_client(client, archive):
323
response = client.get_messages('baz')
324
self.assertEqual(['bar', 'foo'], sorted(m['message_id'] for m in
325
response['messages']))
326
self.assertIs(None, response['next_memo'])
327
self.assertIs(None, response['previous_memo'])
328
329
def test_get_messages_by_id(self):
330
client = GrackleClient('localhost', 8437)
331
archive = {
332
'baz': [make_message('foo'), make_message('bar')]}
333
with ForkedFakeService.from_client(client, archive):
334
response = client.get_messages('baz', message_ids=['foo'])
335
message, = response['messages']
336
self.assertEqual('foo', message['message_id'])
337
338
def test_get_messages_batching(self):
339
client = GrackleClient('localhost', 8438)
340
archive = {'baz': [make_message('foo'), make_message('bar')]}
341
with ForkedFakeService.from_client(client, archive):
342
response = client.get_messages('baz', limit=1)
343
self.assertEqual(1, len(response['messages']))
344
messages = response['messages']
345
response = client.get_messages(
346
'baz', limit=1, memo=response['next_memo'])
347
self.assertEqual(1, len(response['messages']))
348
messages.extend(response['messages'])
349
self.assertMessageIDs(['foo', 'bar'], messages)
350
351
def get_messages_member_order_test(self, key):
352
client = GrackleClient('localhost', 8439)
353
archive = {
354
'baz': [
355
make_message('foo', headers={key: '2011-03-25'}),
356
make_message('bar', headers={key: '2011-03-24'}),
357
]}
358
with ForkedFakeService.from_client(client, archive):
359
response = client.get_messages('baz')
360
self.assertIDOrder(['foo', 'bar'], response['messages'])
361
response = client.get_messages('baz', order=key)
362
self.assertIDOrder(['bar', 'foo'], response['messages'])
363
364
def test_get_messages_date_order(self):
365
self.get_messages_member_order_test('date')
366
367
def test_get_messages_author_order(self):
368
self.get_messages_member_order_test('author')
369
370
def test_get_messages_subject_order(self):
371
self.get_messages_member_order_test('subject')
372
373
def test_get_messages_thread_subject_order(self):
374
archive = {
375
'baz': [
376
make_message('bar', headers={'subject': 'y'}),
377
make_message('qux', headers={'subject': 'z'}),
378
make_message('foo', headers={'subject': 'x',
379
'in-reply-to': 'qux'}),
380
]}
381
client = GrackleClient('localhost', 8439)
382
with ForkedFakeService.from_client(client, archive):
383
response = client.get_messages('baz')
384
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
385
response = client.get_messages('baz', order='subject')
386
self.assertIDOrder(['foo', 'bar', 'qux'], response['messages'])
387
response = client.get_messages('baz', order='thread_subject')
388
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
389
390
def test_get_messages_thread_oldest_order(self):
391
client = GrackleClient('localhost', 8439)
392
archive = {
393
'baz': [
394
make_message('bar', headers={'date': 'x'}),
395
make_message('qux', headers={'date': 'z'}),
396
make_message('foo', headers={'date': 'y',
397
'in-reply-to': 'qux'}),
398
]}
399
with ForkedFakeService.from_client(client, archive):
400
response = client.get_messages('baz')
401
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
402
response = client.get_messages('baz', order='date')
403
self.assertIDOrder(['bar', 'foo', 'qux'], response['messages'])
404
response = client.get_messages('baz', order='thread_oldest')
405
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
406
407
def test_get_messages_thread_newest_order(self):
408
client = GrackleClient('localhost', 8439)
409
archive = {
410
'baz': [
411
make_message('bar', headers={'date': 'x'}),
412
make_message('qux', headers={'date': 'w'}),
413
make_message('foo', headers={'date': 'y',
414
'in-reply-to': 'bar'}),
415
make_message('baz', headers={'date': 'z',
416
'in-reply-to': 'qux'}),
417
]}
418
with ForkedFakeService.from_client(client, archive):
419
response = client.get_messages('baz', order='date')
420
self.assertIDOrder(
421
['qux', 'bar', 'foo', 'baz'], response['messages'])
422
response = client.get_messages('baz', order='thread_newest')
423
self.assertIDOrder(
424
['bar', 'foo', 'qux', 'baz'], response['messages'])
425
426
def test_get_messages_unsupported_order(self):
427
client = GrackleClient('localhost', 8439)
428
archive = {
429
'baz': [
430
make_message('foo', headers={'date': '2011-03-25'}),
431
make_message('foo', headers={'date': '2011-03-24'}),
432
]}
433
with ForkedFakeService.from_client(client, archive):
434
with ExpectedException(UnsupportedOrder, ''):
435
client.get_messages('baz', order='nonsense')
436
437
def test_get_messages_headers_no_headers(self):
438
client = GrackleClient('localhost', 8440)
439
archive = {'baz': [make_message('foo')]}
440
with ForkedFakeService.from_client(client, archive):
441
response = client.get_messages('baz', headers=[
442
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
443
first_message = response['messages'][0]
444
self.assertEqual('foo', first_message['message_id'])
445
self.assertEqual({}, first_message['headers'])
446
447
def test_get_messages_headers_exclude_headers(self):
448
client = GrackleClient('localhost', 8441)
449
archive = {
450
'baz': [make_message('foo', headers={'From': 'me'})]}
451
with ForkedFakeService.from_client(client, archive):
452
response = client.get_messages('baz', headers=[
453
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
454
first_message = response['messages'][0]
455
self.assertEqual('foo', first_message['message_id'])
456
self.assertEqual({}, first_message['headers'])
457
458
def test_get_messages_headers_include_headers(self):
459
client = GrackleClient('localhost', 8442)
460
archive = {
461
'baz': [
462
make_message('foo', headers={'From': 'me', 'To': 'you'})]}
463
with ForkedFakeService.from_client(client, archive):
464
response = client.get_messages('baz', headers=[
465
'From', 'To'])
466
first_message = response['messages'][0]
467
self.assertEqual('foo', first_message['message_id'])
468
self.assertEqual({'From': 'me', 'To': 'you'}, first_message['headers'])
469
470
def test_get_messages_max_body_length(self):
471
client = GrackleClient('localhost', 8443)
472
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
473
with ForkedFakeService.from_client(client, archive):
474
response = client.get_messages('baz', max_body_length=3)
475
first_message = response['messages'][0]
476
self.assertEqual('abc', first_message['body'])
477
478
def test_include_hidden(self):
479
client = GrackleClient('localhost', 8444)
480
archive = {
481
'baz': [
482
make_message('foo', hidden=True),
483
make_message('bar', hidden=False),
484
]}
485
with ForkedFakeService.from_client(client, archive):
486
response = client.get_messages('baz', include_hidden=True)
487
self.assertMessageIDs(['bar', 'foo'], response['messages'])
488
response = client.get_messages('baz', include_hidden=False)
489
self.assertMessageIDs(['bar'], response['messages'])
490
491
def test_display_type_unknown_value(self):
492
client = GrackleClient('localhost', 8445)
493
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
494
with ForkedFakeService.from_client(client, archive):
495
with ExpectedException(UnsupportedDisplayType, ''):
496
client.get_messages('baz', display_type='unknown')
497
498
def test_display_type_headers_only(self):
499
client = GrackleClient('localhost', 8446)
500
archive = {
501
'baz': [
502
make_message('foo', body=u'abcdefghi',
503
headers={'From': 'me', 'To': 'you'})]}
504
with ForkedFakeService.from_client(client, archive):
505
response = client.get_messages('baz', display_type='headers-only')
506
first_message = response['messages'][0]
507
self.assertEqual('foo', first_message['message_id'])
508
self.assertEqual(
509
{'From': 'me', 'Message-Id': 'foo', 'To': 'you'},
510
first_message['headers'])
511
self.assertNotIn('body', first_message)
512
513
def test_display_type_text_only(self):
514
client = GrackleClient('localhost', 8446)
515
archive = {
516
'baz': [
517
make_mime_message(
518
'foo', 'abcdefghi',
519
headers={'From': 'me', 'To': 'you'},
520
attachment_type='text/x-diff')]}
521
with ForkedFakeService.from_client(client, archive):
522
response = client.get_messages('baz', display_type='text-only')
523
first_message = response['messages'][0]
524
self.assertEqual('foo', first_message['message_id'])
525
self.assertEqual('me', first_message['headers']['From'])
526
self.assertEqual('you', first_message['headers']['To'])
527
self.assertEqual('abcdefghi', first_message['body'])
528
529
def test_display_type_all(self):
530
client = GrackleClient('localhost', 8447)
531
archive = {
532
'baz': [
533
make_mime_message(
534
'foo', 'abcdefghi',
535
headers={'From': 'me', 'To': 'you'},
536
attachment_type='text/x-diff')]}
537
with ForkedFakeService.from_client(client, archive):
538
response = client.get_messages('baz', display_type='all')
539
first_message = response['messages'][0]
540
self.assertEqual('foo', first_message['message_id'])
541
self.assertEqual('me', first_message['headers']['From'])
542
self.assertEqual('you', first_message['headers']['To'])
543
self.assertEqual(
544
'abcdefghi\n\nattactment data.', first_message['body'])
545
546
def test_date_range(self):
547
client = GrackleClient('localhost', 8448)
548
archive = {
549
'baz': [
550
make_mime_message(
551
'foo', 'abcdefghi', headers={'date': '2011-12-31'}),
552
make_mime_message(
553
'bar', 'abcdefghi', headers={'date': '2012-01-01'}),
554
make_mime_message(
555
'qux', 'abcdefghi', headers={'date': '2012-01-15'}),
556
make_mime_message(
557
'naf', 'abcdefghi', headers={'date': '2012-01-31'}),
558
make_mime_message(
559
'doh', 'abcdefghi', headers={'date': '2012-02-01'}),
560
]}
561
with ForkedFakeService.from_client(client, archive):
562
response = client.get_messages(
563
'baz', date_range='2012-01-01..2012-01-31')
564
ids = sorted(m['message_id'] for m in response['messages'])
565
self.assertEqual(['bar', 'naf', 'qux'], ids)
566
567
def test_date_range_unparsabledaterange(self):
568
client = GrackleClient('localhost', 8448)
569
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
570
with ForkedFakeService.from_client(client, archive):
571
with ExpectedException(UnparsableDateRange, ''):
572
client.get_messages('baz', date_range='2012-01-01')
573
574
def test_date_range_unparsabledaterange_missing_part(self):
575
client = GrackleClient('localhost', 8448)
576
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
577
with ForkedFakeService.from_client(client, archive):
578
with ExpectedException(UnparsableDateRange, ''):
579
client.get_messages('baz', date_range='2012-01-01..')
Loggerhead is a web-based interface for Breezy
Version: 2.0.1