« back to all changes in this revision
Viewing changes to grackle/tests/test_client.py
- browse files at revision 11
- compare with another revision
- download diff
- download tarball
- view history from revision 11
- Committer: Aaron Bentley
- Date: 2012-01-11 11:18:15 UTC
- Revision ID: aaron@canonical.com-20120111111815-0ecv9fihdao2u68o
Start working on GET.
- files modified:
- grackle-put-message
added
removed
grackle/tests/test_client.py
2
2
HTTPServer,
3
3
BaseHTTPRequestHandler,
4
4
)
5
from email.message import Message
6
from email.mime.multipart import MIMEMultipart
7
from email.mime.text import MIMEText
8
5
import httplib
9
import logging
10
6
import os
11
7
from signal import SIGKILL
12
import simplejson
13
8
from StringIO import StringIO
14
import sys
15
9
from unittest import TestCase
16
from urlparse import urlparse
17
from urlparse import parse_qs
18
10
19
11
from testtools import ExpectedException
20
12
21
13
from grackle.client import (
22
14
GrackleClient,
23
UnparsableDateRange,
24
UnsupportedDisplayType,
25
UnsupportedOrder,
26
15
)
27
16
28
17
29
def make_message(message_id, body='body', headers=None, hidden=False):
30
if headers is None:
31
headers = {}
32
headers['Message-Id'] = message_id
33
message = {
34
'message_id': message_id,
35
'headers': headers,
36
'thread_id': message_id,
37
'date': headers.get('date', '2005-01-01'),
38
'subject': headers.get('subject', 'subject'),
39
'author': headers.get('author', 'author'),
40
'hidden': hidden,
41
'attachments': [],
42
'replies': headers.get('in-reply-to', None),
43
'body': body,
44
}
45
return message
46
47
48
def make_mime_message(message_id, body='body', headers=None, hidden=False,
49
attachment_type=None):
50
message = MIMEMultipart()
51
message.attach(MIMEText(body))
52
if attachment_type is not None:
53
attachment = Message()
54
attachment.set_payload('attactment data.')
55
attachment['Content-Type'] = attachment_type
56
attachment['Content-Disposition'] = 'attachment; filename="file.ext"'
57
message.attach(attachment)
58
return make_message(message_id, message.get_payload(), headers, hidden)
59
60
61
def threaded_messages(messages):
62
threads = {}
63
count = 0
64
pending = []
65
for message in messages:
66
if message.get('replies') is None:
67
threads[message['message_id']] = [message]
68
count += 1
69
else:
70
pending.append(message)
71
for message in pending:
72
threads[message['replies']].append(message)
73
return threads.values()
74
75
76
class GrackleStore:
77
"""A memory-backed message store."""
78
79
def __init__(self, messages):
80
"""Constructor."""
81
self.messages = messages
82
83
@staticmethod
84
def is_multipart(message):
85
return isinstance(message['body'], list)
86
87
def get_messages(self, archive_id, query_string):
88
"""Return matching messages.
89
90
:param archive_id: The archive to retrieve from.
91
:param query_string: Contains 'parameters', which is a JSON-format
92
string describing parameters.
93
"""
94
query = parse_qs(query_string)
95
parameters = simplejson.loads(query['parameters'][0])
96
order = parameters.get('order')
97
messages = self.messages[archive_id]
98
if order is not None:
99
if order not in SUPPORTED_ORDERS:
100
raise UnsupportedOrder
101
elif order.startswith('thread_'):
102
threaded = threaded_messages(messages)
103
messages = []
104
if order == 'thread_subject':
105
threaded.sort(key=lambda t: t[0]['subject'])
106
if order == 'thread_oldest':
107
threaded.sort(key=lambda t: min(m['date'] for m in t))
108
if order == 'thread_newest':
109
threaded.sort(key=lambda t: max(m['date'] for m in t))
110
for thread in threaded:
111
messages.extend(thread)
112
else:
113
messages.sort(key=lambda m: m[order])
114
display_type = parameters.get('display_type', 'all')
115
if display_type not in SUPPORTED_DISPLAY_TYPES:
116
raise UnsupportedDisplayType
117
if 'date_range' in parameters:
118
try:
119
start_date, end_date = parameters['date_range'].split('..')
120
except ValueError:
121
raise UnparsableDateRange
122
new_messages = []
123
for message in messages:
124
if (not parameters['include_hidden'] and message['hidden']):
125
continue
126
if ('message_ids' in parameters
127
and message['message_id'] not in parameters['message_ids']):
128
continue
129
if ('date_range' in parameters
130
and (message['date'] < start_date
131
or message['date'] > end_date)):
132
continue
133
message = dict(message)
134
if 'headers' in parameters:
135
headers = dict(
136
(k, v) for k, v in message['headers'].iteritems()
137
if k in parameters['headers'])
138
message['headers'] = headers
139
if display_type == 'headers-only':
140
del message['body']
141
elif display_type == 'text-only' and self.is_multipart(message):
142
text_parts = [
143
part.get_payload() for part in message['body']
144
if part.get_content_type() == 'text/plain']
145
message['body'] = '\n\n'.join(text_parts)
146
elif display_type == 'all' and self.is_multipart(message):
147
parts = [str(part.get_payload()) for part in message['body']]
148
message['body'] = '\n\n'.join(parts)
149
max_body = parameters.get('max_body_length')
150
if max_body is not None and display_type != 'headers-only':
151
message['body'] = message['body'][:max_body]
152
new_messages.append(message)
153
messages = new_messages
154
limit = parameters.get('limit', 100)
155
memo = parameters.get('memo')
156
message_id_indices = dict(
157
(m['message_id'], idx) for idx, m in enumerate(messages))
158
if memo is None:
159
start = 0
160
else:
161
start = message_id_indices[memo.encode('rot13')]
162
if start > 0:
163
previous_memo = messages[start - 1]['message_id'].encode('rot13')
164
else:
165
previous_memo = None
166
end = min(start + limit, len(messages))
167
if end < len(messages):
168
next_memo = messages[end]['message_id'].encode('rot13')
169
else:
170
next_memo = None
171
messages = messages[start:end]
172
173
response = {
174
'messages': messages,
175
'next_memo': next_memo,
176
'previous_memo': previous_memo
177
}
178
return response
179
180
181
class ForkedFakeService:
182
"""A Grackle service fake, as a ContextManager."""
183
184
def __init__(self, port, messages=None, write_logs=False):
185
"""Constructor.
186
187
:param port: The tcp port to use.
188
:param messages: A dict of lists of dicts representing messages. The
189
outer dict represents the archive, the list represents the list of
190
messages for that archive.
191
:param write_logs: If true, log messages will be written to stdout.
192
"""
18
class Forked:
19
20
def __init__(self, func_or_method, *args):
21
self.func_or_method = func_or_method
193
22
self.pid = None
194
self.port = port
195
if messages is None:
196
self.messages = {}
197
else:
198
self.messages = messages
199
self.read_end, self.write_end = os.pipe()
200
self.write_logs = write_logs
201
202
@staticmethod
203
def from_client(client, messages=None):
204
"""Instantiate a ForkedFakeService from the client.
205
206
:param port: The client to provide service for.
207
:param messages: A dict of lists of dicts representing messages. The
208
outer dict represents the archive, the list represents the list of
209
messages for that archive.
210
"""
211
return ForkedFakeService(client.port, messages)
212
213
def is_ready(self):
214
"""Tell the parent process that the server is ready for writes."""
215
os.write(self.write_end, 'asdf')
23
self.args = args
216
24
217
25
def __enter__(self):
218
"""Run the service.
219
220
Fork and start a server in the child. Return when the server is ready
221
for use."""
222
26
pid = os.fork()
223
if pid == 0:
224
self.start_server()
225
self.pid = pid
226
os.read(self.read_end, 1)
227
return
27
if pid != 0:
28
self.pid = pid
29
return
30
self.func_or_method(*self.args)
228
31
229
def start_server(self):
230
"""Start the HTTP server."""
231
service = HTTPServer(('', self.port), FakeGrackleRequestHandler)
232
service.store = GrackleStore(self.messages)
233
for archive_id, messages in service.store.messages.iteritems():
234
for message in messages:
235
message.setdefault('headers', {})
236
self.is_ready()
237
if self.write_logs:
238
logging.basicConfig(
239
stream=sys.stderr, level=logging.INFO)
240
service.serve_forever()
241
32
242
33
def __exit__(self, exc_type, exc_val, traceback):
243
34
os.kill(self.pid, SIGKILL)
244
35
245
36
246
SUPPORTED_DISPLAY_TYPES = set(['all', 'text-only', 'headers-only'])
247
248
249
SUPPORTED_ORDERS = set(
250
['date', 'author', 'subject', 'thread_newest', 'thread_oldest',
251
'thread_subject'])
252
253
254
37
class FakeGrackleRequestHandler(BaseHTTPRequestHandler):
255
"""A request handler that forwards to server.store."""
256
257
def __init__(self, *args, **kwargs):
258
"""Constructor. Sets up logging."""
259
self.logger = logging.getLogger('http')
260
BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
261
38
262
39
def do_POST(self):
263
"""Create a message on POST."""
264
40
message = self.rfile.read(int(self.headers['content-length']))
265
41
if message == 'This is a message':
266
42
self.send_response(httplib.CREATED)
269
45
else:
270
46
self.send_error(httplib.BAD_REQUEST)
271
47
272
def do_GET(self):
273
"""Retrieve a list of messages on GET."""
274
scheme, netloc, path, params, query_string, fragments = (
275
urlparse(self.path))
276
parts = path.split('/')
277
if parts[1] == 'archive':
278
try:
279
response = self.server.store.get_messages(
280
parts[2], query_string)
281
self.send_response(httplib.OK)
282
self.end_headers()
283
self.wfile.write(simplejson.dumps(response))
284
except Exception, error:
285
self.send_response(
286
httplib.BAD_REQUEST, error.__doc__)
287
return
288
289
def log_message(self, format, *args):
290
"""Override log_message to use standard Python logging."""
291
message = "%s - - [%s] %s\n" % (
292
self.address_string(), self.log_date_time_string(), format % args)
293
self.logger.info(message)
48
49
def run_service(port, messages):
50
service = HTTPServer(('', port), FakeGrackleRequestHandler)
51
service.messages = messages
52
service.serve_forever()
53
54
55
def fake_grackle_service(client, messages=None):
56
if messages is None:
57
messages = {}
58
return Forked(run_service, client.port, messages)
294
59
295
60
296
61
class TestPutMessage(TestCase):
297
62
298
63
def test_put_message(self):
299
client = GrackleClient('localhost', 8436)
300
with ForkedFakeService.from_client(client):
64
client = GrackleClient('localhost', 8435)
65
with fake_grackle_service(client):
66
import time; time.sleep(0.5)
301
67
client.put_message('arch1', 'asdf', StringIO('This is a message'))
302
68
with ExpectedException(Exception, 'wtf'):
303
69
client.put_message('arch1', 'asdf',
306
72
307
73
class TestGetMessages(TestCase):
308
74
309
def assertIDOrder(self, ids, messages):
310
self.assertEqual(ids, [m['message_id'] for m in messages])
311
312
def assertMessageIDs(self, ids, messages):
313
self.assertIDOrder(
314
sorted(ids), sorted(messages, key=lambda m: m['message_id']))
315
316
75
def test_get_messages(self):
317
76
client = GrackleClient('localhost', 8435)
318
archive = {
319
'baz': [make_message('foo'), make_message('bar')]}
320
with ForkedFakeService.from_client(client, archive):
321
response = client.get_messages('baz')
322
self.assertEqual(['bar', 'foo'], sorted(m['message_id'] for m in
323
response['messages']))
324
self.assertIs(None, response['next_memo'])
325
self.assertIs(None, response['previous_memo'])
326
327
def test_get_messages_by_id(self):
328
client = GrackleClient('localhost', 8437)
329
archive = {
330
'baz': [make_message('foo'), make_message('bar')]}
331
with ForkedFakeService.from_client(client, archive):
332
response = client.get_messages('baz', message_ids=['foo'])
333
message, = response['messages']
334
self.assertEqual('foo', message['message_id'])
335
336
def test_get_messages_batching(self):
337
client = GrackleClient('localhost', 8438)
338
archive = {'baz': [make_message('foo'), make_message('bar')]}
339
with ForkedFakeService.from_client(client, archive):
340
response = client.get_messages('baz', limit=1)
341
self.assertEqual(1, len(response['messages']))
342
messages = response['messages']
343
response = client.get_messages(
344
'baz', limit=1, memo=response['next_memo'])
345
self.assertEqual(1, len(response['messages']))
346
messages.extend(response['messages'])
347
self.assertMessageIDs(['foo', 'bar'], messages)
348
349
def get_messages_member_order_test(self, key):
350
client = GrackleClient('localhost', 8439)
351
archive = {
352
'baz': [
353
make_message('foo', headers={key: '2011-03-25'}),
354
make_message('bar', headers={key: '2011-03-24'}),
355
]}
356
with ForkedFakeService.from_client(client, archive):
357
response = client.get_messages('baz')
358
self.assertIDOrder(['foo', 'bar'], response['messages'])
359
response = client.get_messages('baz', order=key)
360
self.assertIDOrder(['bar', 'foo'], response['messages'])
361
362
def test_get_messages_date_order(self):
363
self.get_messages_member_order_test('date')
364
365
def test_get_messages_author_order(self):
366
self.get_messages_member_order_test('author')
367
368
def test_get_messages_subject_order(self):
369
self.get_messages_member_order_test('subject')
370
371
def test_get_messages_thread_subject_order(self):
372
archive = {
373
'baz': [
374
make_message('bar', headers={'subject': 'y'}),
375
make_message('qux', headers={'subject': 'z'}),
376
make_message('foo', headers={'subject': 'x',
377
'in-reply-to': 'qux'}),
378
]}
379
client = GrackleClient('localhost', 8439)
380
with ForkedFakeService.from_client(client, archive):
381
response = client.get_messages('baz')
382
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
383
response = client.get_messages('baz', order='subject')
384
self.assertIDOrder(['foo', 'bar', 'qux'], response['messages'])
385
response = client.get_messages('baz', order='thread_subject')
386
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
387
388
def test_get_messages_thread_oldest_order(self):
389
client = GrackleClient('localhost', 8439)
390
archive = {
391
'baz': [
392
make_message('bar', headers={'date': 'x'}),
393
make_message('qux', headers={'date': 'z'}),
394
make_message('foo', headers={'date': 'y',
395
'in-reply-to': 'qux'}),
396
]}
397
with ForkedFakeService.from_client(client, archive):
398
response = client.get_messages('baz')
399
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
400
response = client.get_messages('baz', order='date')
401
self.assertIDOrder(['bar', 'foo', 'qux'], response['messages'])
402
response = client.get_messages('baz', order='thread_oldest')
403
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
404
405
def test_get_messages_thread_newest_order(self):
406
client = GrackleClient('localhost', 8439)
407
archive = {
408
'baz': [
409
make_message('bar', headers={'date': 'x'}),
410
make_message('qux', headers={'date': 'w'}),
411
make_message('foo', headers={'date': 'y',
412
'in-reply-to': 'bar'}),
413
make_message('baz', headers={'date': 'z',
414
'in-reply-to': 'qux'}),
415
]}
416
with ForkedFakeService.from_client(client, archive):
417
response = client.get_messages('baz', order='date')
418
self.assertIDOrder(
419
['qux', 'bar', 'foo', 'baz'], response['messages'])
420
response = client.get_messages('baz', order='thread_newest')
421
self.assertIDOrder(
422
['bar', 'foo', 'qux', 'baz'], response['messages'])
423
424
def test_get_messages_unsupported_order(self):
425
client = GrackleClient('localhost', 8439)
426
archive = {
427
'baz': [
428
make_message('foo', headers={'date': '2011-03-25'}),
429
make_message('foo', headers={'date': '2011-03-24'}),
430
]}
431
with ForkedFakeService.from_client(client, archive):
432
with ExpectedException(UnsupportedOrder, ''):
433
client.get_messages('baz', order='nonsense')
434
435
def test_get_messages_headers_no_headers(self):
436
client = GrackleClient('localhost', 8440)
437
archive = {'baz': [make_message('foo')]}
438
with ForkedFakeService.from_client(client, archive):
439
response = client.get_messages('baz', headers=[
440
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
441
first_message = response['messages'][0]
442
self.assertEqual('foo', first_message['message_id'])
443
self.assertEqual({}, first_message['headers'])
444
445
def test_get_messages_headers_exclude_headers(self):
446
client = GrackleClient('localhost', 8441)
447
archive = {
448
'baz': [make_message('foo', headers={'From': 'me'})]}
449
with ForkedFakeService.from_client(client, archive):
450
response = client.get_messages('baz', headers=[
451
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
452
first_message = response['messages'][0]
453
self.assertEqual('foo', first_message['message_id'])
454
self.assertEqual({}, first_message['headers'])
455
456
def test_get_messages_headers_include_headers(self):
457
client = GrackleClient('localhost', 8442)
458
archive = {
459
'baz': [
460
make_message('foo', headers={'From': 'me', 'To': 'you'})]}
461
with ForkedFakeService.from_client(client, archive):
462
response = client.get_messages('baz', headers=[
463
'From', 'To'])
464
first_message = response['messages'][0]
465
self.assertEqual('foo', first_message['message_id'])
466
self.assertEqual({'From': 'me', 'To': 'you'}, first_message['headers'])
467
468
def test_get_messages_max_body_length(self):
469
client = GrackleClient('localhost', 8443)
470
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
471
with ForkedFakeService.from_client(client, archive):
472
response = client.get_messages('baz', max_body_length=3)
473
first_message = response['messages'][0]
474
self.assertEqual('abc', first_message['body'])
475
476
def test_include_hidden(self):
477
client = GrackleClient('localhost', 8444)
478
archive = {
479
'baz': [
480
make_message('foo', hidden=True),
481
make_message('bar', hidden=False),
482
]}
483
with ForkedFakeService.from_client(client, archive):
484
response = client.get_messages('baz', include_hidden=True)
485
self.assertMessageIDs(['bar', 'foo'], response['messages'])
486
response = client.get_messages('baz', include_hidden=False)
487
self.assertMessageIDs(['bar'], response['messages'])
488
489
def test_display_type_unknown_value(self):
490
client = GrackleClient('localhost', 8445)
491
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
492
with ForkedFakeService.from_client(client, archive):
493
with ExpectedException(UnsupportedDisplayType, ''):
494
client.get_messages('baz', display_type='unknown')
495
496
def test_display_type_headers_only(self):
497
client = GrackleClient('localhost', 8446)
498
archive = {
499
'baz': [
500
make_message('foo', body=u'abcdefghi',
501
headers={'From': 'me', 'To': 'you'})]}
502
with ForkedFakeService.from_client(client, archive):
503
response = client.get_messages('baz', display_type='headers-only')
504
first_message = response['messages'][0]
505
self.assertEqual('foo', first_message['message_id'])
506
self.assertEqual(
507
{'From': 'me', 'Message-Id': 'foo', 'To': 'you'},
508
first_message['headers'])
509
self.assertNotIn('body', first_message)
510
511
def test_display_type_text_only(self):
512
client = GrackleClient('localhost', 8446)
513
archive = {
514
'baz': [
515
make_mime_message(
516
'foo', 'abcdefghi',
517
headers={'From': 'me', 'To': 'you'},
518
attachment_type='text/x-diff')]}
519
with ForkedFakeService.from_client(client, archive):
520
response = client.get_messages('baz', display_type='text-only')
521
first_message = response['messages'][0]
522
self.assertEqual('foo', first_message['message_id'])
523
self.assertEqual('me', first_message['headers']['From'])
524
self.assertEqual('you', first_message['headers']['To'])
525
self.assertEqual('abcdefghi', first_message['body'])
526
527
def test_display_type_all(self):
528
client = GrackleClient('localhost', 8447)
529
archive = {
530
'baz': [
531
make_mime_message(
532
'foo', 'abcdefghi',
533
headers={'From': 'me', 'To': 'you'},
534
attachment_type='text/x-diff')]}
535
with ForkedFakeService.from_client(client, archive):
536
response = client.get_messages('baz', display_type='all')
537
first_message = response['messages'][0]
538
self.assertEqual('foo', first_message['message_id'])
539
self.assertEqual('me', first_message['headers']['From'])
540
self.assertEqual('you', first_message['headers']['To'])
541
self.assertEqual(
542
'abcdefghi\n\nattactment data.', first_message['body'])
543
544
def test_date_range(self):
545
client = GrackleClient('localhost', 8448)
546
archive = {
547
'baz': [
548
make_mime_message(
549
'foo', 'abcdefghi', headers={'date': '2011-12-31'}),
550
make_mime_message(
551
'bar', 'abcdefghi', headers={'date': '2012-01-01'}),
552
make_mime_message(
553
'qux', 'abcdefghi', headers={'date': '2012-01-15'}),
554
make_mime_message(
555
'naf', 'abcdefghi', headers={'date': '2012-01-31'}),
556
make_mime_message(
557
'doh', 'abcdefghi', headers={'date': '2012-02-01'}),
558
]}
559
with ForkedFakeService.from_client(client, archive):
560
response = client.get_messages(
561
'baz', date_range='2012-01-01..2012-01-31')
562
ids = sorted(m['message_id'] for m in response['messages'])
563
self.assertEqual(['bar', 'naf', 'qux'], ids)
564
565
def test_date_range_unparsabledaterange(self):
566
client = GrackleClient('localhost', 8448)
567
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
568
with ForkedFakeService.from_client(client, archive):
569
with ExpectedException(UnparsableDateRange, ''):
570
client.get_messages('baz', date_range='2012-01-01')
77
with fake_grackle_service(client,
78
{'baz':
79
[{'message-id': 'foo'},
80
{'message-id': 'bar'}]}):
81
response = client.get_messages('baz')
82
self.assertEqual(['bar', 'foo'], sorted(response.keys()))
Loggerhead is a web-based interface for Breezy
Version: 2.0.1