3
3
BaseHTTPRequestHandler,
5
from email.message import Message
6
from email.mime.multipart import MIMEMultipart
7
from email.mime.text import MIMEText
11
7
from signal import SIGKILL
13
8
from StringIO import StringIO
15
9
from unittest import TestCase
16
from urlparse import urlparse
17
from urlparse import parse_qs
19
11
from testtools import ExpectedException
21
13
from grackle.client import (
24
UnsupportedDisplayType,
29
def make_message(message_id, body='body', headers=None, hidden=False):
32
headers['Message-Id'] = message_id
34
'message_id': message_id,
36
'thread_id': message_id,
37
'date': headers.get('date', '2005-01-01'),
38
'subject': headers.get('subject', 'subject'),
39
'author': headers.get('author', 'author'),
42
'replies': headers.get('in-reply-to', None),
48
def make_mime_message(message_id, body='body', headers=None, hidden=False,
49
attachment_type=None):
50
message = MIMEMultipart()
51
message.attach(MIMEText(body))
52
if attachment_type is not None:
53
attachment = Message()
54
attachment.set_payload('attactment data.')
55
attachment['Content-Type'] = attachment_type
56
attachment['Content-Disposition'] = 'attachment; filename="file.ext"'
57
message.attach(attachment)
58
return make_message(message_id, message.get_payload(), headers, hidden)
61
def threaded_messages(messages):
65
for message in messages:
66
if message.get('replies') is None:
67
threads[message['message_id']] = [message]
70
pending.append(message)
71
for message in pending:
72
threads[message['replies']].append(message)
73
return threads.values()
77
"""A memory-backed message store."""
79
def __init__(self, messages):
81
self.messages = messages
84
def is_multipart(message):
85
return isinstance(message['body'], list)
87
def get_messages(self, archive_id, query_string):
88
"""Return matching messages.
90
:param archive_id: The archive to retrieve from.
91
:param query_string: Contains 'parameters', which is a JSON-format
92
string describing parameters.
94
query = parse_qs(query_string)
95
parameters = simplejson.loads(query['parameters'][0])
96
order = parameters.get('order')
97
messages = self.messages[archive_id]
99
if order not in SUPPORTED_ORDERS:
100
raise UnsupportedOrder
101
elif order.startswith('thread_'):
102
threaded = threaded_messages(messages)
104
if order == 'thread_subject':
105
threaded.sort(key=lambda t: t[0]['subject'])
106
if order == 'thread_oldest':
107
threaded.sort(key=lambda t: min(m['date'] for m in t))
108
if order == 'thread_newest':
109
threaded.sort(key=lambda t: max(m['date'] for m in t))
110
for thread in threaded:
111
messages.extend(thread)
113
messages.sort(key=lambda m: m[order])
114
display_type = parameters.get('display_type', 'all')
115
if display_type not in SUPPORTED_DISPLAY_TYPES:
116
raise UnsupportedDisplayType
117
if 'date_range' in parameters:
119
start_date, end_date = parameters['date_range'].split('..')
120
if not start_date or not end_date:
121
raise UnparsableDateRange
123
raise UnparsableDateRange
125
for message in messages:
126
if (not parameters['include_hidden'] and message['hidden']):
128
if ('message_ids' in parameters
129
and message['message_id'] not in parameters['message_ids']):
131
if ('date_range' in parameters
132
and (message['date'] < start_date
133
or message['date'] > end_date)):
135
message = dict(message)
136
if 'headers' in parameters:
138
(k, v) for k, v in message['headers'].iteritems()
139
if k in parameters['headers'])
140
message['headers'] = headers
141
if display_type == 'headers-only':
143
elif display_type == 'text-only' and self.is_multipart(message):
145
part.get_payload() for part in message['body']
146
if part.get_content_type() == 'text/plain']
147
message['body'] = '\n\n'.join(text_parts)
148
elif display_type == 'all' and self.is_multipart(message):
149
parts = [str(part.get_payload()) for part in message['body']]
150
message['body'] = '\n\n'.join(parts)
151
max_body = parameters.get('max_body_length')
152
if max_body is not None and display_type != 'headers-only':
153
message['body'] = message['body'][:max_body]
154
new_messages.append(message)
155
messages = new_messages
156
limit = parameters.get('limit', 100)
157
memo = parameters.get('memo')
158
message_id_indices = dict(
159
(m['message_id'], idx) for idx, m in enumerate(messages))
163
start = message_id_indices[memo.encode('rot13')]
165
previous_memo = messages[start - 1]['message_id'].encode('rot13')
168
end = min(start + limit, len(messages))
169
if end < len(messages):
170
next_memo = messages[end]['message_id'].encode('rot13')
173
messages = messages[start:end]
176
'messages': messages,
177
'next_memo': next_memo,
178
'previous_memo': previous_memo
183
class ForkedFakeService:
184
"""A Grackle service fake, as a ContextManager."""
186
def __init__(self, port, messages=None, write_logs=False):
189
:param port: The tcp port to use.
190
:param messages: A dict of lists of dicts representing messages. The
191
outer dict represents the archive, the list represents the list of
192
messages for that archive.
193
:param write_logs: If true, log messages will be written to stdout.
20
def __init__(self, port, messages=None):
200
self.messages = messages
23
self.messages = messages
201
24
self.read_end, self.write_end = os.pipe()
202
self.write_logs = write_logs
205
def from_client(client, messages=None):
206
"""Instantiate a ForkedFakeService from the client.
208
:param port: The client to provide service for.
209
:param messages: A dict of lists of dicts representing messages. The
210
outer dict represents the archive, the list represents the list of
211
messages for that archive.
213
return ForkedFakeService(client.port, messages)
215
26
def is_ready(self):
216
"""Tell the parent process that the server is ready for writes."""
217
27
os.write(self.write_end, 'asdf')
219
29
def __enter__(self):
222
Fork and start a server in the child. Return when the server is ready
226
32
self.start_server()
309
76
class TestGetMessages(TestCase):
311
def assertIDOrder(self, ids, messages):
312
self.assertEqual(ids, [m['message_id'] for m in messages])
314
def assertMessageIDs(self, ids, messages):
316
sorted(ids), sorted(messages, key=lambda m: m['message_id']))
318
78
def test_get_messages(self):
319
79
client = GrackleClient('localhost', 8435)
321
'baz': [make_message('foo'), make_message('bar')]}
322
with ForkedFakeService.from_client(client, archive):
323
response = client.get_messages('baz')
324
self.assertEqual(['bar', 'foo'], sorted(m['message_id'] for m in
325
response['messages']))
326
self.assertIs(None, response['next_memo'])
327
self.assertIs(None, response['previous_memo'])
329
def test_get_messages_by_id(self):
330
client = GrackleClient('localhost', 8437)
332
'baz': [make_message('foo'), make_message('bar')]}
333
with ForkedFakeService.from_client(client, archive):
334
response = client.get_messages('baz', message_ids=['foo'])
335
message, = response['messages']
336
self.assertEqual('foo', message['message_id'])
338
def test_get_messages_batching(self):
339
client = GrackleClient('localhost', 8438)
340
archive = {'baz': [make_message('foo'), make_message('bar')]}
341
with ForkedFakeService.from_client(client, archive):
342
response = client.get_messages('baz', limit=1)
343
self.assertEqual(1, len(response['messages']))
344
messages = response['messages']
345
response = client.get_messages(
346
'baz', limit=1, memo=response['next_memo'])
347
self.assertEqual(1, len(response['messages']))
348
messages.extend(response['messages'])
349
self.assertMessageIDs(['foo', 'bar'], messages)
351
def get_messages_member_order_test(self, key):
352
client = GrackleClient('localhost', 8439)
355
make_message('foo', headers={key: '2011-03-25'}),
356
make_message('bar', headers={key: '2011-03-24'}),
358
with ForkedFakeService.from_client(client, archive):
359
response = client.get_messages('baz')
360
self.assertIDOrder(['foo', 'bar'], response['messages'])
361
response = client.get_messages('baz', order=key)
362
self.assertIDOrder(['bar', 'foo'], response['messages'])
364
def test_get_messages_date_order(self):
365
self.get_messages_member_order_test('date')
367
def test_get_messages_author_order(self):
368
self.get_messages_member_order_test('author')
370
def test_get_messages_subject_order(self):
371
self.get_messages_member_order_test('subject')
373
def test_get_messages_thread_subject_order(self):
376
make_message('bar', headers={'subject': 'y'}),
377
make_message('qux', headers={'subject': 'z'}),
378
make_message('foo', headers={'subject': 'x',
379
'in-reply-to': 'qux'}),
381
client = GrackleClient('localhost', 8439)
382
with ForkedFakeService.from_client(client, archive):
383
response = client.get_messages('baz')
384
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
385
response = client.get_messages('baz', order='subject')
386
self.assertIDOrder(['foo', 'bar', 'qux'], response['messages'])
387
response = client.get_messages('baz', order='thread_subject')
388
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
390
def test_get_messages_thread_oldest_order(self):
391
client = GrackleClient('localhost', 8439)
394
make_message('bar', headers={'date': 'x'}),
395
make_message('qux', headers={'date': 'z'}),
396
make_message('foo', headers={'date': 'y',
397
'in-reply-to': 'qux'}),
399
with ForkedFakeService.from_client(client, archive):
400
response = client.get_messages('baz')
401
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
402
response = client.get_messages('baz', order='date')
403
self.assertIDOrder(['bar', 'foo', 'qux'], response['messages'])
404
response = client.get_messages('baz', order='thread_oldest')
405
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
407
def test_get_messages_thread_newest_order(self):
408
client = GrackleClient('localhost', 8439)
411
make_message('bar', headers={'date': 'x'}),
412
make_message('qux', headers={'date': 'w'}),
413
make_message('foo', headers={'date': 'y',
414
'in-reply-to': 'bar'}),
415
make_message('baz', headers={'date': 'z',
416
'in-reply-to': 'qux'}),
418
with ForkedFakeService.from_client(client, archive):
419
response = client.get_messages('baz', order='date')
421
['qux', 'bar', 'foo', 'baz'], response['messages'])
422
response = client.get_messages('baz', order='thread_newest')
424
['bar', 'foo', 'qux', 'baz'], response['messages'])
426
def test_get_messages_unsupported_order(self):
427
client = GrackleClient('localhost', 8439)
430
make_message('foo', headers={'date': '2011-03-25'}),
431
make_message('foo', headers={'date': '2011-03-24'}),
433
with ForkedFakeService.from_client(client, archive):
434
with ExpectedException(UnsupportedOrder, ''):
435
client.get_messages('baz', order='nonsense')
437
def test_get_messages_headers_no_headers(self):
438
client = GrackleClient('localhost', 8440)
439
archive = {'baz': [make_message('foo')]}
440
with ForkedFakeService.from_client(client, archive):
441
response = client.get_messages('baz', headers=[
442
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
443
first_message = response['messages'][0]
444
self.assertEqual('foo', first_message['message_id'])
445
self.assertEqual({}, first_message['headers'])
447
def test_get_messages_headers_exclude_headers(self):
448
client = GrackleClient('localhost', 8441)
450
'baz': [make_message('foo', headers={'From': 'me'})]}
451
with ForkedFakeService.from_client(client, archive):
452
response = client.get_messages('baz', headers=[
453
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
454
first_message = response['messages'][0]
455
self.assertEqual('foo', first_message['message_id'])
456
self.assertEqual({}, first_message['headers'])
458
def test_get_messages_headers_include_headers(self):
459
client = GrackleClient('localhost', 8442)
462
make_message('foo', headers={'From': 'me', 'To': 'you'})]}
463
with ForkedFakeService.from_client(client, archive):
464
response = client.get_messages('baz', headers=[
466
first_message = response['messages'][0]
467
self.assertEqual('foo', first_message['message_id'])
468
self.assertEqual({'From': 'me', 'To': 'you'}, first_message['headers'])
470
def test_get_messages_max_body_length(self):
471
client = GrackleClient('localhost', 8443)
472
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
473
with ForkedFakeService.from_client(client, archive):
474
response = client.get_messages('baz', max_body_length=3)
475
first_message = response['messages'][0]
476
self.assertEqual('abc', first_message['body'])
478
def test_include_hidden(self):
479
client = GrackleClient('localhost', 8444)
482
make_message('foo', hidden=True),
483
make_message('bar', hidden=False),
485
with ForkedFakeService.from_client(client, archive):
486
response = client.get_messages('baz', include_hidden=True)
487
self.assertMessageIDs(['bar', 'foo'], response['messages'])
488
response = client.get_messages('baz', include_hidden=False)
489
self.assertMessageIDs(['bar'], response['messages'])
491
def test_display_type_unknown_value(self):
492
client = GrackleClient('localhost', 8445)
493
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
494
with ForkedFakeService.from_client(client, archive):
495
with ExpectedException(UnsupportedDisplayType, ''):
496
client.get_messages('baz', display_type='unknown')
498
def test_display_type_headers_only(self):
499
client = GrackleClient('localhost', 8446)
502
make_message('foo', body=u'abcdefghi',
503
headers={'From': 'me', 'To': 'you'})]}
504
with ForkedFakeService.from_client(client, archive):
505
response = client.get_messages('baz', display_type='headers-only')
506
first_message = response['messages'][0]
507
self.assertEqual('foo', first_message['message_id'])
509
{'From': 'me', 'Message-Id': 'foo', 'To': 'you'},
510
first_message['headers'])
511
self.assertNotIn('body', first_message)
513
def test_display_type_text_only(self):
514
client = GrackleClient('localhost', 8446)
519
headers={'From': 'me', 'To': 'you'},
520
attachment_type='text/x-diff')]}
521
with ForkedFakeService.from_client(client, archive):
522
response = client.get_messages('baz', display_type='text-only')
523
first_message = response['messages'][0]
524
self.assertEqual('foo', first_message['message_id'])
525
self.assertEqual('me', first_message['headers']['From'])
526
self.assertEqual('you', first_message['headers']['To'])
527
self.assertEqual('abcdefghi', first_message['body'])
529
def test_display_type_all(self):
530
client = GrackleClient('localhost', 8447)
535
headers={'From': 'me', 'To': 'you'},
536
attachment_type='text/x-diff')]}
537
with ForkedFakeService.from_client(client, archive):
538
response = client.get_messages('baz', display_type='all')
539
first_message = response['messages'][0]
540
self.assertEqual('foo', first_message['message_id'])
541
self.assertEqual('me', first_message['headers']['From'])
542
self.assertEqual('you', first_message['headers']['To'])
544
'abcdefghi\n\nattactment data.', first_message['body'])
546
def test_date_range(self):
547
client = GrackleClient('localhost', 8448)
551
'foo', 'abcdefghi', headers={'date': '2011-12-31'}),
553
'bar', 'abcdefghi', headers={'date': '2012-01-01'}),
555
'qux', 'abcdefghi', headers={'date': '2012-01-15'}),
557
'naf', 'abcdefghi', headers={'date': '2012-01-31'}),
559
'doh', 'abcdefghi', headers={'date': '2012-02-01'}),
561
with ForkedFakeService.from_client(client, archive):
562
response = client.get_messages(
563
'baz', date_range='2012-01-01..2012-01-31')
564
ids = sorted(m['message_id'] for m in response['messages'])
565
self.assertEqual(['bar', 'naf', 'qux'], ids)
567
def test_date_range_unparsabledaterange(self):
568
client = GrackleClient('localhost', 8449)
569
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
570
with ForkedFakeService.from_client(client, archive):
571
with ExpectedException(UnparsableDateRange, ''):
572
client.get_messages('baz', date_range='2012-01-01')
574
def test_date_range_unparsabledaterange_missing_part(self):
575
client = GrackleClient('localhost', 8450)
576
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
577
with ForkedFakeService.from_client(client, archive):
578
with ExpectedException(UnparsableDateRange, ''):
579
client.get_messages('baz', date_range='2012-01-01..')
581
def test_date_range_unparsabledaterange_extra_part(self):
582
client = GrackleClient('localhost', 8451)
583
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
584
with ForkedFakeService.from_client(client, archive):
585
with ExpectedException(UnparsableDateRange, ''):
586
client.get_messages('baz', date_range='2012-01..12-02..12-03')
80
with fake_grackle_service(client,
82
[{'message-id': 'foo'},
83
{'message-id': 'bar'}]}):
84
response = client.get_messages('baz')
85
self.assertEqual(['bar', 'foo'], sorted(response.keys()))