3
3
BaseHTTPRequestHandler,
5
from email.message import Message
6
from email.mime.multipart import MIMEMultipart
7
from email.mime.text import MIMEText
11
7
from signal import SIGKILL
13
8
from StringIO import StringIO
15
9
from unittest import TestCase
16
from urlparse import urlparse
17
from urlparse import parse_qs
19
11
from testtools import ExpectedException
21
13
from grackle.client import (
23
UnsupportedDisplayType,
28
def make_message(message_id, body='body', headers=None, hidden=False):
31
headers['Message-Id'] = message_id
33
'message_id': message_id,
35
'thread_id': message_id,
36
'date': headers.get('date', '2005-01-01'),
37
'subject': headers.get('subject', 'subject'),
38
'author': headers.get('author', 'author'),
41
'replies': headers.get('in-reply-to', None),
47
def make_mime_message(message_id, body='body', headers=None, hidden=False,
48
attachment_type=None):
49
message = MIMEMultipart()
50
message.attach(MIMEText(body))
51
if attachment_type is not None:
52
attachment = Message()
53
attachment.set_payload('attactment data.')
54
attachment['Content-Type'] = attachment_type
55
attachment['Content-Disposition'] = 'attachment; filename="file.ext"'
56
message.attach(attachment)
57
return make_message(message_id, message.get_payload(), headers, hidden)
60
def threaded_messages(messages):
64
for message in messages:
65
if message.get('replies') is None:
66
threads[message['message_id']] = [message]
69
pending.append(message)
70
for message in pending:
71
threads[message['replies']].append(message)
72
return threads.values()
76
"""A memory-backed message store."""
78
def __init__(self, messages):
80
self.messages = messages
83
def is_multipart(message):
84
return isinstance(message['body'], list)
86
def get_messages(self, archive_id, query_string):
87
"""Return matching messages.
89
:param archive_id: The archive to retrieve from.
90
:param query_string: Contains 'parameters', which is a JSON-format
91
string describing parameters.
93
query = parse_qs(query_string)
94
parameters = simplejson.loads(query['parameters'][0])
95
order = parameters.get('order')
96
messages = self.messages[archive_id]
98
if order not in SUPPORTED_ORDERS:
99
raise UnsupportedOrder
100
elif order.startswith('thread_'):
101
threaded = threaded_messages(messages)
103
if order == 'thread_subject':
104
threaded.sort(key=lambda t: t[0]['subject'])
105
if order == 'thread_oldest':
106
threaded.sort(key=lambda t: min(m['date'] for m in t))
107
if order == 'thread_newest':
108
threaded.sort(key=lambda t: max(m['date'] for m in t))
109
for thread in threaded:
110
messages.extend(thread)
112
messages.sort(key=lambda m: m[order])
113
display_type = parameters.get('display_type', 'all')
114
if display_type not in SUPPORTED_DISPLAY_TYPES:
115
raise UnsupportedDisplayType
117
for message in messages:
118
if (not parameters['include_hidden'] and message['hidden']):
120
if ('message_ids' in parameters
121
and message['message_id'] not in parameters['message_ids']):
123
message = dict(message)
124
if 'headers' in parameters:
126
(k, v) for k, v in message['headers'].iteritems()
127
if k in parameters['headers'])
128
message['headers'] = headers
129
if display_type == 'headers-only':
131
elif display_type == 'text-only' and self.is_multipart(message):
133
part.get_payload() for part in message['body']
134
if part.get_content_type() == 'text/plain']
135
message['body'] = '\n\n'.join(text_parts)
136
elif display_type == 'all' and self.is_multipart(message):
137
parts = [str(part.get_payload()) for part in message['body']]
138
message['body'] = '\n\n'.join(parts)
139
max_body = parameters.get('max_body_length')
140
if max_body is not None and display_type != 'headers-only':
141
message['body'] = message['body'][:max_body]
142
new_messages.append(message)
143
messages = new_messages
144
limit = parameters.get('limit', 100)
145
memo = parameters.get('memo')
146
message_id_indices = dict(
147
(m['message_id'], idx) for idx, m in enumerate(messages))
151
start = message_id_indices[memo.encode('rot13')]
153
previous_memo = messages[start - 1]['message_id'].encode('rot13')
156
end = min(start + limit, len(messages))
157
if end < len(messages):
158
next_memo = messages[end]['message_id'].encode('rot13')
161
messages = messages[start:end]
164
'messages': messages,
165
'next_memo': next_memo,
166
'previous_memo': previous_memo
171
class ForkedFakeService:
172
"""A Grackle service fake, as a ContextManager."""
174
def __init__(self, port, messages=None, write_logs=False):
177
:param port: The tcp port to use.
178
:param messages: A dict of lists of dicts representing messages. The
179
outer dict represents the archive, the list represents the list of
180
messages for that archive.
181
:param write_logs: If true, log messages will be written to stdout.
20
def __init__(self, func_or_method, *args):
21
self.func_or_method = func_or_method
188
self.messages = messages
189
self.read_end, self.write_end = os.pipe()
190
self.write_logs = write_logs
193
def from_client(client, messages=None):
194
"""Instantiate a ForkedFakeService from the client.
196
:param port: The client to provide service for.
197
:param messages: A dict of lists of dicts representing messages. The
198
outer dict represents the archive, the list represents the list of
199
messages for that archive.
201
return ForkedFakeService(client.port, messages)
204
"""Tell the parent process that the server is ready for writes."""
205
os.write(self.write_end, 'asdf')
207
25
def __enter__(self):
210
Fork and start a server in the child. Return when the server is ready
216
os.read(self.read_end, 1)
30
self.func_or_method(*self.args)
219
def start_server(self):
220
"""Start the HTTP server."""
221
service = HTTPServer(('', self.port), FakeGrackleRequestHandler)
222
service.store = GrackleStore(self.messages)
223
for archive_id, messages in service.store.messages.iteritems():
224
for message in messages:
225
message.setdefault('headers', {})
229
stream=sys.stderr, level=logging.INFO)
230
service.serve_forever()
232
33
def __exit__(self, exc_type, exc_val, traceback):
233
34
os.kill(self.pid, SIGKILL)
236
SUPPORTED_DISPLAY_TYPES = set(['all', 'text-only', 'headers-only'])
239
SUPPORTED_ORDERS = set(
240
['date', 'author', 'subject', 'thread_newest', 'thread_oldest',
244
37
class FakeGrackleRequestHandler(BaseHTTPRequestHandler):
245
"""A request handler that forwards to server.store."""
247
def __init__(self, *args, **kwargs):
248
"""Constructor. Sets up logging."""
249
self.logger = logging.getLogger('http')
250
BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
253
"""Create a message on POST."""
254
40
message = self.rfile.read(int(self.headers['content-length']))
255
41
if message == 'This is a message':
256
42
self.send_response(httplib.CREATED)
260
46
self.send_error(httplib.BAD_REQUEST)
263
"""Retrieve a list of messages on GET."""
264
scheme, netloc, path, params, query_string, fragments = (
266
parts = path.split('/')
267
if parts[1] == 'archive':
269
response = self.server.store.get_messages(
270
parts[2], query_string)
271
self.send_response(httplib.OK)
273
self.wfile.write(simplejson.dumps(response))
274
except UnsupportedOrder:
276
httplib.BAD_REQUEST, UnsupportedOrder.__doc__)
278
except UnsupportedDisplayType:
280
httplib.BAD_REQUEST, UnsupportedDisplayType.__doc__)
283
def log_message(self, format, *args):
284
"""Override log_message to use standard Python logging."""
285
message = "%s - - [%s] %s\n" % (
286
self.address_string(), self.log_date_time_string(), format % args)
287
self.logger.info(message)
49
def run_service(port):
50
service = HTTPServer(('', port), FakeGrackleRequestHandler)
51
service.serve_forever()
290
55
class TestPutMessage(TestCase):
292
57
def test_put_message(self):
293
client = GrackleClient('localhost', 8436)
294
with ForkedFakeService.from_client(client):
58
client = GrackleClient('localhost', 8435)
59
with Forked(run_service, client.port):
295
60
client.put_message('arch1', 'asdf', StringIO('This is a message'))
296
61
with ExpectedException(Exception, 'wtf'):
297
62
client.put_message('arch1', 'asdf',
298
63
StringIO('This is not a message'))
301
class TestGetMessages(TestCase):
303
def assertIDOrder(self, ids, messages):
304
self.assertEqual(ids, [m['message_id'] for m in messages])
306
def assertMessageIDs(self, ids, messages):
308
sorted(ids), sorted(messages, key=lambda m: m['message_id']))
310
def test_get_messages(self):
311
client = GrackleClient('localhost', 8435)
313
'baz': [make_message('foo'), make_message('bar')]}
314
with ForkedFakeService.from_client(client, archive):
315
response = client.get_messages('baz')
316
self.assertEqual(['bar', 'foo'], sorted(m['message_id'] for m in
317
response['messages']))
318
self.assertIs(None, response['next_memo'])
319
self.assertIs(None, response['previous_memo'])
321
def test_get_messages_by_id(self):
322
client = GrackleClient('localhost', 8437)
324
'baz': [make_message('foo'), make_message('bar')]}
325
with ForkedFakeService.from_client(client, archive):
326
response = client.get_messages('baz', message_ids=['foo'])
327
message, = response['messages']
328
self.assertEqual('foo', message['message_id'])
330
def test_get_messages_batching(self):
331
client = GrackleClient('localhost', 8438)
332
archive = {'baz': [make_message('foo'), make_message('bar')]}
333
with ForkedFakeService.from_client(client, archive):
334
response = client.get_messages('baz', limit=1)
335
self.assertEqual(1, len(response['messages']))
336
messages = response['messages']
337
response = client.get_messages(
338
'baz', limit=1, memo=response['next_memo'])
339
self.assertEqual(1, len(response['messages']))
340
messages.extend(response['messages'])
341
self.assertMessageIDs(['foo', 'bar'], messages)
343
def get_messages_member_order_test(self, key):
344
client = GrackleClient('localhost', 8439)
347
make_message('foo', headers={key: '2011-03-25'}),
348
make_message('bar', headers={key: '2011-03-24'}),
350
with ForkedFakeService.from_client(client, archive):
351
response = client.get_messages('baz')
352
self.assertIDOrder(['foo', 'bar'], response['messages'])
353
response = client.get_messages('baz', order=key)
354
self.assertIDOrder(['bar', 'foo'], response['messages'])
356
def test_get_messages_date_order(self):
357
self.get_messages_member_order_test('date')
359
def test_get_messages_author_order(self):
360
self.get_messages_member_order_test('author')
362
def test_get_messages_subject_order(self):
363
self.get_messages_member_order_test('subject')
365
def test_get_messages_thread_subject_order(self):
368
make_message('bar', headers={'subject': 'y'}),
369
make_message('qux', headers={'subject': 'z'}),
370
make_message('foo', headers={'subject': 'x',
371
'in-reply-to': 'qux'}),
373
client = GrackleClient('localhost', 8439)
374
with ForkedFakeService.from_client(client, archive):
375
response = client.get_messages('baz')
376
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
377
response = client.get_messages('baz', order='subject')
378
self.assertIDOrder(['foo', 'bar', 'qux'], response['messages'])
379
response = client.get_messages('baz', order='thread_subject')
380
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
382
def test_get_messages_thread_oldest_order(self):
383
client = GrackleClient('localhost', 8439)
386
make_message('bar', headers={'date': 'x'}),
387
make_message('qux', headers={'date': 'z'}),
388
make_message('foo', headers={'date': 'y',
389
'in-reply-to': 'qux'}),
391
with ForkedFakeService.from_client(client, archive):
392
response = client.get_messages('baz')
393
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
394
response = client.get_messages('baz', order='date')
395
self.assertIDOrder(['bar', 'foo', 'qux'], response['messages'])
396
response = client.get_messages('baz', order='thread_oldest')
397
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
399
def test_get_messages_thread_newest_order(self):
400
client = GrackleClient('localhost', 8439)
403
make_message('bar', headers={'date': 'x'}),
404
make_message('qux', headers={'date': 'w'}),
405
make_message('foo', headers={'date': 'y',
406
'in-reply-to': 'bar'}),
407
make_message('baz', headers={'date': 'z',
408
'in-reply-to': 'qux'}),
410
with ForkedFakeService.from_client(client, archive):
411
response = client.get_messages('baz', order='date')
413
['qux', 'bar', 'foo', 'baz'], response['messages'])
414
response = client.get_messages('baz', order='thread_newest')
416
['bar', 'foo', 'qux', 'baz'], response['messages'])
418
def test_get_messages_unsupported_order(self):
419
client = GrackleClient('localhost', 8439)
422
make_message('foo', headers={'date': '2011-03-25'}),
423
make_message('foo', headers={'date': '2011-03-24'}),
425
with ForkedFakeService.from_client(client, archive):
426
with ExpectedException(UnsupportedOrder, ''):
427
client.get_messages('baz', order='nonsense')
429
def test_get_messages_headers_no_headers(self):
430
client = GrackleClient('localhost', 8440)
431
archive = {'baz': [make_message('foo')]}
432
with ForkedFakeService.from_client(client, archive):
433
response = client.get_messages('baz', headers=[
434
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
435
first_message = response['messages'][0]
436
self.assertEqual('foo', first_message['message_id'])
437
self.assertEqual({}, first_message['headers'])
439
def test_get_messages_headers_exclude_headers(self):
440
client = GrackleClient('localhost', 8441)
442
'baz': [make_message('foo', headers={'From': 'me'})]}
443
with ForkedFakeService.from_client(client, archive):
444
response = client.get_messages('baz', headers=[
445
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
446
first_message = response['messages'][0]
447
self.assertEqual('foo', first_message['message_id'])
448
self.assertEqual({}, first_message['headers'])
450
def test_get_messages_headers_include_headers(self):
451
client = GrackleClient('localhost', 8442)
454
make_message('foo', headers={'From': 'me', 'To': 'you'})]}
455
with ForkedFakeService.from_client(client, archive):
456
response = client.get_messages('baz', headers=[
458
first_message = response['messages'][0]
459
self.assertEqual('foo', first_message['message_id'])
460
self.assertEqual({'From': 'me', 'To': 'you'}, first_message['headers'])
462
def test_get_messages_max_body_length(self):
463
client = GrackleClient('localhost', 8443)
464
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
465
with ForkedFakeService.from_client(client, archive):
466
response = client.get_messages('baz', max_body_length=3)
467
first_message = response['messages'][0]
468
self.assertEqual('abc', first_message['body'])
470
def test_include_hidden(self):
471
client = GrackleClient('localhost', 8444)
474
make_message('foo', hidden=True),
475
make_message('bar', hidden=False),
477
with ForkedFakeService.from_client(client, archive):
478
response = client.get_messages('baz', include_hidden=True)
479
self.assertMessageIDs(['bar', 'foo'], response['messages'])
480
response = client.get_messages('baz', include_hidden=False)
481
self.assertMessageIDs(['bar'], response['messages'])
483
def test_display_type_unknown_value(self):
484
client = GrackleClient('localhost', 8445)
485
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
486
with ForkedFakeService.from_client(client, archive):
487
with ExpectedException(UnsupportedDisplayType, ''):
488
client.get_messages('baz', display_type='unknown')
490
def test_display_type_headers_only(self):
491
client = GrackleClient('localhost', 8446)
494
make_message('foo', body=u'abcdefghi',
495
headers={'From': 'me', 'To': 'you'})]}
496
with ForkedFakeService.from_client(client, archive):
497
response = client.get_messages('baz', display_type='headers-only')
498
first_message = response['messages'][0]
499
self.assertEqual('foo', first_message['message_id'])
501
{'From': 'me', 'Message-Id': 'foo', 'To': 'you'},
502
first_message['headers'])
503
self.assertNotIn('body', first_message)
505
def test_display_type_text_only(self):
506
client = GrackleClient('localhost', 8446)
511
headers={'From': 'me', 'To': 'you'},
512
attachment_type='text/x-diff')]}
513
with ForkedFakeService.from_client(client, archive):
514
response = client.get_messages('baz', display_type='text-only')
515
first_message = response['messages'][0]
516
self.assertEqual('foo', first_message['message_id'])
517
self.assertEqual('me', first_message['headers']['From'])
518
self.assertEqual('you', first_message['headers']['To'])
519
self.assertEqual('abcdefghi', first_message['body'])
521
def test_display_type_all(self):
522
client = GrackleClient('localhost', 8447)
527
headers={'From': 'me', 'To': 'you'},
528
attachment_type='text/x-diff')]}
529
with ForkedFakeService.from_client(client, archive):
530
response = client.get_messages('baz', display_type='all')
531
first_message = response['messages'][0]
532
self.assertEqual('foo', first_message['message_id'])
533
self.assertEqual('me', first_message['headers']['From'])
534
self.assertEqual('you', first_message['headers']['To'])
536
'abcdefghi\n\nattactment data.', first_message['body'])