1
from BaseHTTPServer import (
3
BaseHTTPRequestHandler,
5
from email.message import Message
6
from email.mime.multipart import MIMEMultipart
7
from email.mime.text import MIMEText
11
from signal import SIGKILL
13
from StringIO import StringIO
15
from unittest import TestCase
16
from urlparse import urlparse
17
from urlparse import parse_qs
19
from testtools import ExpectedException
21
from grackle.client import (
24
UnsupportedDisplayType,
29
def make_message(message_id, body='body', headers=None, hidden=False):
32
headers['Message-Id'] = message_id
34
'message_id': message_id,
36
'thread_id': message_id,
37
'date': headers.get('date', '2005-01-01'),
38
'subject': headers.get('subject', 'subject'),
39
'author': headers.get('author', 'author'),
42
'replies': headers.get('in-reply-to', None),
48
def make_mime_message(message_id, body='body', headers=None, hidden=False,
49
attachment_type=None):
50
message = MIMEMultipart()
51
message.attach(MIMEText(body))
52
if attachment_type is not None:
53
attachment = Message()
54
attachment.set_payload('attactment data.')
55
attachment['Content-Type'] = attachment_type
56
attachment['Content-Disposition'] = 'attachment; filename="file.ext"'
57
message.attach(attachment)
58
return make_message(message_id, message.get_payload(), headers, hidden)
61
def threaded_messages(messages):
65
for message in messages:
66
if message.get('replies') is None:
67
threads[message['message_id']] = [message]
70
pending.append(message)
71
for message in pending:
72
threads[message['replies']].append(message)
73
return threads.values()
77
"""A memory-backed message store."""
79
def __init__(self, messages):
81
self.messages = messages
84
def is_multipart(message):
85
return isinstance(message['body'], list)
87
def get_messages(self, archive_id, query_string):
88
"""Return matching messages.
90
:param archive_id: The archive to retrieve from.
91
:param query_string: Contains 'parameters', which is a JSON-format
92
string describing parameters.
94
query = parse_qs(query_string)
95
parameters = simplejson.loads(query['parameters'][0])
96
order = parameters.get('order')
97
messages = self.messages[archive_id]
99
if order not in SUPPORTED_ORDERS:
100
raise UnsupportedOrder
101
elif order.startswith('thread_'):
102
threaded = threaded_messages(messages)
104
if order == 'thread_subject':
105
threaded.sort(key=lambda t: t[0]['subject'])
106
if order == 'thread_oldest':
107
threaded.sort(key=lambda t: min(m['date'] for m in t))
108
if order == 'thread_newest':
109
threaded.sort(key=lambda t: max(m['date'] for m in t))
110
for thread in threaded:
111
messages.extend(thread)
113
messages.sort(key=lambda m: m[order])
114
display_type = parameters.get('display_type', 'all')
115
if display_type not in SUPPORTED_DISPLAY_TYPES:
116
raise UnsupportedDisplayType
117
if 'date_range' in parameters:
119
start_date, end_date = parameters['date_range'].split('..')
121
raise UnparsableDateRange()
123
for message in messages:
124
if (not parameters['include_hidden'] and message['hidden']):
126
if ('message_ids' in parameters
127
and message['message_id'] not in parameters['message_ids']):
129
if ('date_range' in parameters
130
and (message['date'] < start_date
131
or message['date'] > end_date)):
133
message = dict(message)
134
if 'headers' in parameters:
136
(k, v) for k, v in message['headers'].iteritems()
137
if k in parameters['headers'])
138
message['headers'] = headers
139
if display_type == 'headers-only':
141
elif display_type == 'text-only' and self.is_multipart(message):
143
part.get_payload() for part in message['body']
144
if part.get_content_type() == 'text/plain']
145
message['body'] = '\n\n'.join(text_parts)
146
elif display_type == 'all' and self.is_multipart(message):
147
parts = [str(part.get_payload()) for part in message['body']]
148
message['body'] = '\n\n'.join(parts)
149
max_body = parameters.get('max_body_length')
150
if max_body is not None and display_type != 'headers-only':
151
message['body'] = message['body'][:max_body]
152
new_messages.append(message)
153
messages = new_messages
154
limit = parameters.get('limit', 100)
155
memo = parameters.get('memo')
156
message_id_indices = dict(
157
(m['message_id'], idx) for idx, m in enumerate(messages))
161
start = message_id_indices[memo.encode('rot13')]
163
previous_memo = messages[start - 1]['message_id'].encode('rot13')
166
end = min(start + limit, len(messages))
167
if end < len(messages):
168
next_memo = messages[end]['message_id'].encode('rot13')
171
messages = messages[start:end]
174
'messages': messages,
175
'next_memo': next_memo,
176
'previous_memo': previous_memo
181
class ForkedFakeService:
182
"""A Grackle service fake, as a ContextManager."""
184
def __init__(self, port, messages=None, write_logs=False):
187
:param port: The tcp port to use.
188
:param messages: A dict of lists of dicts representing messages. The
189
outer dict represents the archive, the list represents the list of
190
messages for that archive.
191
:param write_logs: If true, log messages will be written to stdout.
198
self.messages = messages
199
self.read_end, self.write_end = os.pipe()
200
self.write_logs = write_logs
203
def from_client(client, messages=None):
204
"""Instantiate a ForkedFakeService from the client.
206
:param port: The client to provide service for.
207
:param messages: A dict of lists of dicts representing messages. The
208
outer dict represents the archive, the list represents the list of
209
messages for that archive.
211
return ForkedFakeService(client.port, messages)
214
"""Tell the parent process that the server is ready for writes."""
215
os.write(self.write_end, 'asdf')
220
Fork and start a server in the child. Return when the server is ready
226
os.read(self.read_end, 1)
229
def start_server(self):
230
"""Start the HTTP server."""
231
service = HTTPServer(('', self.port), FakeGrackleRequestHandler)
232
service.store = GrackleStore(self.messages)
233
for archive_id, messages in service.store.messages.iteritems():
234
for message in messages:
235
message.setdefault('headers', {})
239
stream=sys.stderr, level=logging.INFO)
240
service.serve_forever()
242
def __exit__(self, exc_type, exc_val, traceback):
243
os.kill(self.pid, SIGKILL)
246
SUPPORTED_DISPLAY_TYPES = set(['all', 'text-only', 'headers-only'])
249
SUPPORTED_ORDERS = set(
250
['date', 'author', 'subject', 'thread_newest', 'thread_oldest',
254
class FakeGrackleRequestHandler(BaseHTTPRequestHandler):
255
"""A request handler that forwards to server.store."""
257
def __init__(self, *args, **kwargs):
258
"""Constructor. Sets up logging."""
259
self.logger = logging.getLogger('http')
260
BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
263
"""Create a message on POST."""
264
message = self.rfile.read(int(self.headers['content-length']))
265
if message == 'This is a message':
266
self.send_response(httplib.CREATED)
270
self.send_error(httplib.BAD_REQUEST)
273
"""Retrieve a list of messages on GET."""
274
scheme, netloc, path, params, query_string, fragments = (
276
parts = path.split('/')
277
if parts[1] == 'archive':
279
response = self.server.store.get_messages(
280
parts[2], query_string)
281
self.send_response(httplib.OK)
283
self.wfile.write(simplejson.dumps(response))
284
except UnsupportedOrder:
286
httplib.BAD_REQUEST, UnsupportedOrder.__doc__)
288
except UnsupportedDisplayType:
290
httplib.BAD_REQUEST, UnsupportedDisplayType.__doc__)
293
def log_message(self, format, *args):
294
"""Override log_message to use standard Python logging."""
295
message = "%s - - [%s] %s\n" % (
296
self.address_string(), self.log_date_time_string(), format % args)
297
self.logger.info(message)
300
class TestPutMessage(TestCase):
302
def test_put_message(self):
303
client = GrackleClient('localhost', 8436)
304
with ForkedFakeService.from_client(client):
305
client.put_message('arch1', 'asdf', StringIO('This is a message'))
306
with ExpectedException(Exception, 'wtf'):
307
client.put_message('arch1', 'asdf',
308
StringIO('This is not a message'))
311
class TestGetMessages(TestCase):
313
def assertIDOrder(self, ids, messages):
314
self.assertEqual(ids, [m['message_id'] for m in messages])
316
def assertMessageIDs(self, ids, messages):
318
sorted(ids), sorted(messages, key=lambda m: m['message_id']))
320
def test_get_messages(self):
321
client = GrackleClient('localhost', 8435)
323
'baz': [make_message('foo'), make_message('bar')]}
324
with ForkedFakeService.from_client(client, archive):
325
response = client.get_messages('baz')
326
self.assertEqual(['bar', 'foo'], sorted(m['message_id'] for m in
327
response['messages']))
328
self.assertIs(None, response['next_memo'])
329
self.assertIs(None, response['previous_memo'])
331
def test_get_messages_by_id(self):
332
client = GrackleClient('localhost', 8437)
334
'baz': [make_message('foo'), make_message('bar')]}
335
with ForkedFakeService.from_client(client, archive):
336
response = client.get_messages('baz', message_ids=['foo'])
337
message, = response['messages']
338
self.assertEqual('foo', message['message_id'])
340
def test_get_messages_batching(self):
341
client = GrackleClient('localhost', 8438)
342
archive = {'baz': [make_message('foo'), make_message('bar')]}
343
with ForkedFakeService.from_client(client, archive):
344
response = client.get_messages('baz', limit=1)
345
self.assertEqual(1, len(response['messages']))
346
messages = response['messages']
347
response = client.get_messages(
348
'baz', limit=1, memo=response['next_memo'])
349
self.assertEqual(1, len(response['messages']))
350
messages.extend(response['messages'])
351
self.assertMessageIDs(['foo', 'bar'], messages)
353
def get_messages_member_order_test(self, key):
354
client = GrackleClient('localhost', 8439)
357
make_message('foo', headers={key: '2011-03-25'}),
358
make_message('bar', headers={key: '2011-03-24'}),
360
with ForkedFakeService.from_client(client, archive):
361
response = client.get_messages('baz')
362
self.assertIDOrder(['foo', 'bar'], response['messages'])
363
response = client.get_messages('baz', order=key)
364
self.assertIDOrder(['bar', 'foo'], response['messages'])
366
def test_get_messages_date_order(self):
367
self.get_messages_member_order_test('date')
369
def test_get_messages_author_order(self):
370
self.get_messages_member_order_test('author')
372
def test_get_messages_subject_order(self):
373
self.get_messages_member_order_test('subject')
375
def test_get_messages_thread_subject_order(self):
378
make_message('bar', headers={'subject': 'y'}),
379
make_message('qux', headers={'subject': 'z'}),
380
make_message('foo', headers={'subject': 'x',
381
'in-reply-to': 'qux'}),
383
client = GrackleClient('localhost', 8439)
384
with ForkedFakeService.from_client(client, archive):
385
response = client.get_messages('baz')
386
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
387
response = client.get_messages('baz', order='subject')
388
self.assertIDOrder(['foo', 'bar', 'qux'], response['messages'])
389
response = client.get_messages('baz', order='thread_subject')
390
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
392
def test_get_messages_thread_oldest_order(self):
393
client = GrackleClient('localhost', 8439)
396
make_message('bar', headers={'date': 'x'}),
397
make_message('qux', headers={'date': 'z'}),
398
make_message('foo', headers={'date': 'y',
399
'in-reply-to': 'qux'}),
401
with ForkedFakeService.from_client(client, archive):
402
response = client.get_messages('baz')
403
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
404
response = client.get_messages('baz', order='date')
405
self.assertIDOrder(['bar', 'foo', 'qux'], response['messages'])
406
response = client.get_messages('baz', order='thread_oldest')
407
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
409
def test_get_messages_thread_newest_order(self):
410
client = GrackleClient('localhost', 8439)
413
make_message('bar', headers={'date': 'x'}),
414
make_message('qux', headers={'date': 'w'}),
415
make_message('foo', headers={'date': 'y',
416
'in-reply-to': 'bar'}),
417
make_message('baz', headers={'date': 'z',
418
'in-reply-to': 'qux'}),
420
with ForkedFakeService.from_client(client, archive):
421
response = client.get_messages('baz', order='date')
423
['qux', 'bar', 'foo', 'baz'], response['messages'])
424
response = client.get_messages('baz', order='thread_newest')
426
['bar', 'foo', 'qux', 'baz'], response['messages'])
428
def test_get_messages_unsupported_order(self):
429
client = GrackleClient('localhost', 8439)
432
make_message('foo', headers={'date': '2011-03-25'}),
433
make_message('foo', headers={'date': '2011-03-24'}),
435
with ForkedFakeService.from_client(client, archive):
436
with ExpectedException(UnsupportedOrder, ''):
437
client.get_messages('baz', order='nonsense')
439
def test_get_messages_headers_no_headers(self):
440
client = GrackleClient('localhost', 8440)
441
archive = {'baz': [make_message('foo')]}
442
with ForkedFakeService.from_client(client, archive):
443
response = client.get_messages('baz', headers=[
444
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
445
first_message = response['messages'][0]
446
self.assertEqual('foo', first_message['message_id'])
447
self.assertEqual({}, first_message['headers'])
449
def test_get_messages_headers_exclude_headers(self):
450
client = GrackleClient('localhost', 8441)
452
'baz': [make_message('foo', headers={'From': 'me'})]}
453
with ForkedFakeService.from_client(client, archive):
454
response = client.get_messages('baz', headers=[
455
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
456
first_message = response['messages'][0]
457
self.assertEqual('foo', first_message['message_id'])
458
self.assertEqual({}, first_message['headers'])
460
def test_get_messages_headers_include_headers(self):
461
client = GrackleClient('localhost', 8442)
464
make_message('foo', headers={'From': 'me', 'To': 'you'})]}
465
with ForkedFakeService.from_client(client, archive):
466
response = client.get_messages('baz', headers=[
468
first_message = response['messages'][0]
469
self.assertEqual('foo', first_message['message_id'])
470
self.assertEqual({'From': 'me', 'To': 'you'}, first_message['headers'])
472
def test_get_messages_max_body_length(self):
473
client = GrackleClient('localhost', 8443)
474
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
475
with ForkedFakeService.from_client(client, archive):
476
response = client.get_messages('baz', max_body_length=3)
477
first_message = response['messages'][0]
478
self.assertEqual('abc', first_message['body'])
480
def test_include_hidden(self):
481
client = GrackleClient('localhost', 8444)
484
make_message('foo', hidden=True),
485
make_message('bar', hidden=False),
487
with ForkedFakeService.from_client(client, archive):
488
response = client.get_messages('baz', include_hidden=True)
489
self.assertMessageIDs(['bar', 'foo'], response['messages'])
490
response = client.get_messages('baz', include_hidden=False)
491
self.assertMessageIDs(['bar'], response['messages'])
493
def test_display_type_unknown_value(self):
494
client = GrackleClient('localhost', 8445)
495
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
496
with ForkedFakeService.from_client(client, archive):
497
with ExpectedException(UnsupportedDisplayType, ''):
498
client.get_messages('baz', display_type='unknown')
500
def test_display_type_headers_only(self):
501
client = GrackleClient('localhost', 8446)
504
make_message('foo', body=u'abcdefghi',
505
headers={'From': 'me', 'To': 'you'})]}
506
with ForkedFakeService.from_client(client, archive):
507
response = client.get_messages('baz', display_type='headers-only')
508
first_message = response['messages'][0]
509
self.assertEqual('foo', first_message['message_id'])
511
{'From': 'me', 'Message-Id': 'foo', 'To': 'you'},
512
first_message['headers'])
513
self.assertNotIn('body', first_message)
515
def test_display_type_text_only(self):
516
client = GrackleClient('localhost', 8446)
521
headers={'From': 'me', 'To': 'you'},
522
attachment_type='text/x-diff')]}
523
with ForkedFakeService.from_client(client, archive):
524
response = client.get_messages('baz', display_type='text-only')
525
first_message = response['messages'][0]
526
self.assertEqual('foo', first_message['message_id'])
527
self.assertEqual('me', first_message['headers']['From'])
528
self.assertEqual('you', first_message['headers']['To'])
529
self.assertEqual('abcdefghi', first_message['body'])
531
def test_display_type_all(self):
532
client = GrackleClient('localhost', 8447)
537
headers={'From': 'me', 'To': 'you'},
538
attachment_type='text/x-diff')]}
539
with ForkedFakeService.from_client(client, archive):
540
response = client.get_messages('baz', display_type='all')
541
first_message = response['messages'][0]
542
self.assertEqual('foo', first_message['message_id'])
543
self.assertEqual('me', first_message['headers']['From'])
544
self.assertEqual('you', first_message['headers']['To'])
546
'abcdefghi\n\nattactment data.', first_message['body'])
548
def test_date_range(self):
549
client = GrackleClient('localhost', 8448)
553
'foo', 'abcdefghi', headers={'date': '2011-12-31'}),
555
'bar', 'abcdefghi', headers={'date': '2012-01-01'}),
557
'qux', 'abcdefghi', headers={'date': '2012-01-15'}),
559
'naf', 'abcdefghi', headers={'date': '2012-01-31'}),
561
'doh', 'abcdefghi', headers={'date': '2012-02-01'}),
563
with ForkedFakeService.from_client(client, archive):
564
response = client.get_messages(
565
'baz', date_range='2012-01-01..2012-01-31')
566
ids = sorted(m['message_id'] for m in response['messages'])
567
self.assertEqual(['bar', 'naf', 'qux'], ids)