« back to all changes in this revision
Viewing changes to grackle/tests/test_client.py
- browse files at revision 24
- compare with another revision
- download diff
- download tarball
- view history from revision 24
- Committer: Aaron Bentley
- Date: 2012-01-13 11:46:14 UTC
- Revision ID: aaron@canonical.com-20120113114614-lw5t8rcn4cidlb3o
Support thread_newest threading.
- files modified:
- grackle-put-message
added
removed
grackle/tests/test_client.py
2
2
HTTPServer,
3
3
BaseHTTPRequestHandler,
4
4
)
5
from email.message import Message
6
from email.mime.multipart import MIMEMultipart
7
from email.mime.text import MIMEText
8
5
import httplib
9
import logging
10
6
import os
11
7
from signal import SIGKILL
12
8
import simplejson
13
9
from StringIO import StringIO
14
import sys
15
10
from unittest import TestCase
16
11
from urlparse import urlparse
17
12
from urlparse import parse_qs
20
15
21
16
from grackle.client import (
22
17
GrackleClient,
23
UnsupportedDisplayType,
24
18
UnsupportedOrder,
25
19
)
26
20
27
21
28
def make_message(message_id, body='body', headers=None, hidden=False):
29
if headers is None:
30
headers = {}
31
headers['Message-Id'] = message_id
32
message = {
33
'message_id': message_id,
34
'headers': headers,
35
'thread_id': message_id,
36
'date': headers.get('date', '2005-01-01'),
37
'subject': headers.get('subject', 'subject'),
38
'author': headers.get('author', 'author'),
39
'hidden': hidden,
40
'attachments': [],
41
'replies': headers.get('in-reply-to', None),
42
'body': body,
43
}
44
return message
45
46
47
def make_mime_message(message_id, body='body', headers=None, hidden=False,
48
attachment_type=None):
49
message = MIMEMultipart()
50
message.attach(MIMEText(body))
51
if attachment_type is not None:
52
attachment = Message()
53
attachment.set_payload('attactment data.')
54
attachment['Content-Type'] = attachment_type
55
attachment['Content-Disposition'] = 'attachment; filename="file.ext"'
56
message.attach(attachment)
57
return make_message(message_id, message.get_payload(), headers, hidden)
58
59
60
22
def threaded_messages(messages):
61
23
threads = {}
62
24
count = 0
63
25
pending = []
64
26
for message in messages:
65
if message.get('replies') is None:
27
if message.get('in_reply_to') is None:
66
28
threads[message['message_id']] = [message]
67
29
count += 1
68
30
else:
69
31
pending.append(message)
70
32
for message in pending:
71
threads[message['replies']].append(message)
33
threads[message['in_reply_to']].append(message)
72
34
return threads.values()
73
35
74
36
75
class GrackleStore:
76
"""A memory-backed message store."""
37
class ForkedFake:
77
38
78
def __init__(self, messages):
79
"""Constructor."""
39
def __init__(self, port, messages=None):
40
self.pid = None
41
self.port = port
80
42
self.messages = messages
81
82
@staticmethod
83
def is_multipart(message):
84
return isinstance(message['body'], list)
85
86
def get_messages(self, archive_id, query_string):
87
"""Return matching messages.
88
89
:param archive_id: The archive to retrieve from.
90
:param query_string: Contains 'parameters', which is a JSON-format
91
string describing parameters.
92
"""
43
self.read_end, self.write_end = os.pipe()
44
45
def is_ready(self):
46
os.write(self.write_end, 'asdf')
47
48
def __enter__(self):
49
pid = os.fork()
50
if pid == 0:
51
self.start_server()
52
self.pid = pid
53
os.read(self.read_end, 1)
54
return
55
56
def start_server(self):
57
service = HTTPServer(('', self.port), FakeGrackleRequestHandler)
58
service.messages = self.messages
59
self.is_ready()
60
service.serve_forever()
61
62
def __exit__(self, exc_type, exc_val, traceback):
63
os.kill(self.pid, SIGKILL)
64
65
66
SUPPORTED_ORDERS = set(
67
['date', 'author', 'subject', 'thread_newest', 'thread_oldest',
68
'thread_subject'])
69
70
71
class FakeGrackleRequestHandler(BaseHTTPRequestHandler):
72
73
def do_POST(self):
74
message = self.rfile.read(int(self.headers['content-length']))
75
if message == 'This is a message':
76
self.send_response(httplib.CREATED)
77
self.end_headers()
78
self.wfile.close()
79
else:
80
self.send_error(httplib.BAD_REQUEST)
81
82
def do_GET(self):
83
scheme, netloc, path, params, query_string, fragments = (
84
urlparse(self.path))
85
archive = os.path.split(path)[1]
93
86
query = parse_qs(query_string)
94
87
parameters = simplejson.loads(query['parameters'][0])
95
88
order = parameters.get('order')
96
messages = self.messages[archive_id]
97
if order is not None:
89
messages = self.server.messages[archive]
90
if order is not None :
98
91
if order not in SUPPORTED_ORDERS:
99
raise UnsupportedOrder
92
self.send_response(httplib.BAD_REQUEST)
93
self.wfile.write('Unsupported order')
94
return
100
95
elif order.startswith('thread_'):
101
96
threaded = threaded_messages(messages)
102
97
messages = []
110
105
messages.extend(thread)
111
106
else:
112
107
messages.sort(key=lambda m: m[order])
113
display_type = parameters.get('display_type', 'all')
114
if display_type not in SUPPORTED_DISPLAY_TYPES:
115
raise UnsupportedDisplayType
116
new_messages = []
117
for message in messages:
118
if (not parameters['include_hidden'] and message['hidden']):
119
continue
120
if ('message_ids' in parameters
121
and message['message_id'] not in parameters['message_ids']):
122
continue
123
message = dict(message)
124
if 'headers' in parameters:
125
headers = dict(
126
(k, v) for k, v in message['headers'].iteritems()
127
if k in parameters['headers'])
128
message['headers'] = headers
129
if display_type == 'headers-only':
130
del message['body']
131
elif display_type == 'text-only' and self.is_multipart(message):
132
text_parts = [
133
part.get_payload() for part in message['body']
134
if part.get_content_type() == 'text/plain']
135
message['body'] = '\n\n'.join(text_parts)
136
elif display_type == 'all' and self.is_multipart(message):
137
parts = [str(part.get_payload()) for part in message['body']]
138
message['body'] = '\n\n'.join(parts)
139
max_body = parameters.get('max_body_length')
140
if max_body is not None and display_type != 'headers-only':
141
message['body'] = message['body'][:max_body]
142
new_messages.append(message)
143
messages = new_messages
108
messages = [m for m in messages
109
if 'message_ids' not in parameters or
110
m['message_id'] in parameters['message_ids']]
111
self.send_response(httplib.OK)
112
self.end_headers()
144
113
limit = parameters.get('limit', 100)
145
114
memo = parameters.get('memo')
146
115
message_id_indices = dict(
159
128
else:
160
129
next_memo = None
161
130
messages = messages[start:end]
162
163
131
response = {
164
132
'messages': messages,
165
133
'next_memo': next_memo,
166
134
'previous_memo': previous_memo
167
135
}
168
return response
169
170
171
class ForkedFakeService:
172
"""A Grackle service fake, as a ContextManager."""
173
174
def __init__(self, port, messages=None, write_logs=False):
175
"""Constructor.
176
177
:param port: The tcp port to use.
178
:param messages: A dict of lists of dicts representing messages. The
179
outer dict represents the archive, the list represents the list of
180
messages for that archive.
181
:param write_logs: If true, log messages will be written to stdout.
182
"""
183
self.pid = None
184
self.port = port
185
if messages is None:
186
self.messages = {}
187
else:
188
self.messages = messages
189
self.read_end, self.write_end = os.pipe()
190
self.write_logs = write_logs
191
192
@staticmethod
193
def from_client(client, messages=None):
194
"""Instantiate a ForkedFakeService from the client.
195
196
:param port: The client to provide service for.
197
:param messages: A dict of lists of dicts representing messages. The
198
outer dict represents the archive, the list represents the list of
199
messages for that archive.
200
"""
201
return ForkedFakeService(client.port, messages)
202
203
def is_ready(self):
204
"""Tell the parent process that the server is ready for writes."""
205
os.write(self.write_end, 'asdf')
206
207
def __enter__(self):
208
"""Run the service.
209
210
Fork and start a server in the child. Return when the server is ready
211
for use."""
212
pid = os.fork()
213
if pid == 0:
214
self.start_server()
215
self.pid = pid
216
os.read(self.read_end, 1)
217
return
218
219
def start_server(self):
220
"""Start the HTTP server."""
221
service = HTTPServer(('', self.port), FakeGrackleRequestHandler)
222
service.store = GrackleStore(self.messages)
223
for archive_id, messages in service.store.messages.iteritems():
224
for message in messages:
225
message.setdefault('headers', {})
226
self.is_ready()
227
if self.write_logs:
228
logging.basicConfig(
229
stream=sys.stderr, level=logging.INFO)
230
service.serve_forever()
231
232
def __exit__(self, exc_type, exc_val, traceback):
233
os.kill(self.pid, SIGKILL)
234
235
236
SUPPORTED_DISPLAY_TYPES = set(['all', 'text-only', 'headers-only'])
237
238
239
SUPPORTED_ORDERS = set(
240
['date', 'author', 'subject', 'thread_newest', 'thread_oldest',
241
'thread_subject'])
242
243
244
class FakeGrackleRequestHandler(BaseHTTPRequestHandler):
245
"""A request handler that forwards to server.store."""
246
247
def __init__(self, *args, **kwargs):
248
"""Constructor. Sets up logging."""
249
self.logger = logging.getLogger('http')
250
BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
251
252
def do_POST(self):
253
"""Create a message on POST."""
254
message = self.rfile.read(int(self.headers['content-length']))
255
if message == 'This is a message':
256
self.send_response(httplib.CREATED)
257
self.end_headers()
258
self.wfile.close()
259
else:
260
self.send_error(httplib.BAD_REQUEST)
261
262
def do_GET(self):
263
"""Retrieve a list of messages on GET."""
264
scheme, netloc, path, params, query_string, fragments = (
265
urlparse(self.path))
266
parts = path.split('/')
267
if parts[1] == 'archive':
268
try:
269
response = self.server.store.get_messages(
270
parts[2], query_string)
271
self.send_response(httplib.OK)
272
self.end_headers()
273
self.wfile.write(simplejson.dumps(response))
274
except UnsupportedOrder:
275
self.send_response(
276
httplib.BAD_REQUEST, UnsupportedOrder.__doc__)
277
return
278
except UnsupportedDisplayType:
279
self.send_response(
280
httplib.BAD_REQUEST, UnsupportedDisplayType.__doc__)
281
return
282
283
def log_message(self, format, *args):
284
"""Override log_message to use standard Python logging."""
285
message = "%s - - [%s] %s\n" % (
286
self.address_string(), self.log_date_time_string(), format % args)
287
self.logger.info(message)
136
self.wfile.write(simplejson.dumps(response))
137
138
139
def fake_grackle_service(client, messages=None):
140
if messages is None:
141
messages = {}
142
return ForkedFake(client.port, messages)
288
143
289
144
290
145
class TestPutMessage(TestCase):
291
146
292
147
def test_put_message(self):
293
148
client = GrackleClient('localhost', 8436)
294
with ForkedFakeService.from_client(client):
149
with fake_grackle_service(client):
295
150
client.put_message('arch1', 'asdf', StringIO('This is a message'))
296
151
with ExpectedException(Exception, 'wtf'):
297
152
client.put_message('arch1', 'asdf',
305
160
306
161
def assertMessageIDs(self, ids, messages):
307
162
self.assertIDOrder(
308
sorted(ids), sorted(messages, key=lambda m: m['message_id']))
163
sorted(ids), sorted(messages, key=lambda m:m['message_id']))
309
164
310
165
def test_get_messages(self):
311
166
client = GrackleClient('localhost', 8435)
312
archive = {
313
'baz': [make_message('foo'), make_message('bar')]}
314
with ForkedFakeService.from_client(client, archive):
167
with fake_grackle_service(client,
168
{'baz':
169
[{'message_id': 'foo'},
170
{'message_id': 'bar'}]}):
315
171
response = client.get_messages('baz')
316
172
self.assertEqual(['bar', 'foo'], sorted(m['message_id'] for m in
317
173
response['messages']))
320
176
321
177
def test_get_messages_by_id(self):
322
178
client = GrackleClient('localhost', 8437)
323
archive = {
324
'baz': [make_message('foo'), make_message('bar')]}
325
with ForkedFakeService.from_client(client, archive):
179
with fake_grackle_service(client,
180
{'baz':
181
[{'message_id': 'foo'},
182
{'message_id': 'bar'}]}):
326
183
response = client.get_messages('baz', message_ids=['foo'])
327
184
message, = response['messages']
328
185
self.assertEqual('foo', message['message_id'])
329
186
330
187
def test_get_messages_batching(self):
331
188
client = GrackleClient('localhost', 8438)
332
archive = {'baz': [make_message('foo'), make_message('bar')]}
333
with ForkedFakeService.from_client(client, archive):
189
with fake_grackle_service(client,
190
{'baz':
191
[{'message_id': 'foo'},
192
{'message_id': 'bar'}]}):
334
193
response = client.get_messages('baz', limit=1)
335
194
self.assertEqual(1, len(response['messages']))
336
195
messages = response['messages']
342
201
343
202
def get_messages_member_order_test(self, key):
344
203
client = GrackleClient('localhost', 8439)
345
archive = {
346
'baz': [
347
make_message('foo', headers={key: '2011-03-25'}),
348
make_message('bar', headers={key: '2011-03-24'}),
349
]}
350
with ForkedFakeService.from_client(client, archive):
204
with fake_grackle_service(client,
205
{'baz': [{'message_id': 'foo', key: '2011-03-25'},
206
{'message_id': 'bar', key: '2011-03-24'}]}):
351
207
response = client.get_messages('baz')
352
208
self.assertIDOrder(['foo', 'bar'], response['messages'])
353
209
response = client.get_messages('baz', order=key)
363
219
self.get_messages_member_order_test('subject')
364
220
365
221
def test_get_messages_thread_subject_order(self):
366
archive = {
367
'baz': [
368
make_message('bar', headers={'subject': 'y'}),
369
make_message('qux', headers={'subject': 'z'}),
370
make_message('foo', headers={'subject': 'x',
371
'in-reply-to': 'qux'}),
372
]}
373
222
client = GrackleClient('localhost', 8439)
374
with ForkedFakeService.from_client(client, archive):
223
with fake_grackle_service(client, {'baz': [
224
{'message_id': 'bar', 'subject': 'y'},
225
{'message_id': 'qux', 'subject': 'z'},
226
{'message_id': 'foo', 'subject': 'x', 'in_reply_to': 'qux'},
227
]}):
375
228
response = client.get_messages('baz')
376
229
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
377
230
response = client.get_messages('baz', order='subject')
381
234
382
235
def test_get_messages_thread_oldest_order(self):
383
236
client = GrackleClient('localhost', 8439)
384
archive = {
385
'baz': [
386
make_message('bar', headers={'date': 'x'}),
387
make_message('qux', headers={'date': 'z'}),
388
make_message('foo', headers={'date': 'y',
389
'in-reply-to': 'qux'}),
390
]}
391
with ForkedFakeService.from_client(client, archive):
237
with fake_grackle_service(client, {'baz': [
238
{'message_id': 'bar', 'date': 'x'},
239
{'message_id': 'qux', 'date': 'z'},
240
{'message_id': 'foo', 'date': 'y', 'in_reply_to': 'qux'},
241
]}):
392
242
response = client.get_messages('baz')
393
243
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
394
244
response = client.get_messages('baz', order='date')
398
248
399
249
def test_get_messages_thread_newest_order(self):
400
250
client = GrackleClient('localhost', 8439)
401
archive = {
402
'baz': [
403
make_message('bar', headers={'date': 'x'}),
404
make_message('qux', headers={'date': 'w'}),
405
make_message('foo', headers={'date': 'y',
406
'in-reply-to': 'bar'}),
407
make_message('baz', headers={'date': 'z',
408
'in-reply-to': 'qux'}),
409
]}
410
with ForkedFakeService.from_client(client, archive):
251
with fake_grackle_service(client, {'baz': [
252
{'message_id': 'bar', 'date': 'x'},
253
{'message_id': 'qux', 'date': 'w'},
254
{'message_id': 'foo', 'date': 'y', 'in_reply_to': 'bar'},
255
{'message_id': 'baz', 'date': 'z', 'in_reply_to': 'qux'},
256
]}):
411
257
response = client.get_messages('baz', order='date')
412
258
self.assertIDOrder(
413
259
['qux', 'bar', 'foo', 'baz'], response['messages'])
417
263
418
264
def test_get_messages_unsupported_order(self):
419
265
client = GrackleClient('localhost', 8439)
420
archive = {
421
'baz': [
422
make_message('foo', headers={'date': '2011-03-25'}),
423
make_message('foo', headers={'date': '2011-03-24'}),
424
]}
425
with ForkedFakeService.from_client(client, archive):
426
with ExpectedException(UnsupportedOrder, ''):
266
with fake_grackle_service(client,
267
{'baz': [{'message_id': 'foo', 'date': '2011-03-25'},
268
{'message_id': 'bar', 'date': '2011-03-24'}]}):
269
with ExpectedException(UnsupportedOrder):
427
270
client.get_messages('baz', order='nonsense')
428
429
def test_get_messages_headers_no_headers(self):
430
client = GrackleClient('localhost', 8440)
431
archive = {'baz': [make_message('foo')]}
432
with ForkedFakeService.from_client(client, archive):
433
response = client.get_messages('baz', headers=[
434
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
435
first_message = response['messages'][0]
436
self.assertEqual('foo', first_message['message_id'])
437
self.assertEqual({}, first_message['headers'])
438
439
def test_get_messages_headers_exclude_headers(self):
440
client = GrackleClient('localhost', 8441)
441
archive = {
442
'baz': [make_message('foo', headers={'From': 'me'})]}
443
with ForkedFakeService.from_client(client, archive):
444
response = client.get_messages('baz', headers=[
445
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
446
first_message = response['messages'][0]
447
self.assertEqual('foo', first_message['message_id'])
448
self.assertEqual({}, first_message['headers'])
449
450
def test_get_messages_headers_include_headers(self):
451
client = GrackleClient('localhost', 8442)
452
archive = {
453
'baz': [
454
make_message('foo', headers={'From': 'me', 'To': 'you'})]}
455
with ForkedFakeService.from_client(client, archive):
456
response = client.get_messages('baz', headers=[
457
'From', 'To'])
458
first_message = response['messages'][0]
459
self.assertEqual('foo', first_message['message_id'])
460
self.assertEqual({'From': 'me', 'To': 'you'}, first_message['headers'])
461
462
def test_get_messages_max_body_length(self):
463
client = GrackleClient('localhost', 8443)
464
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
465
with ForkedFakeService.from_client(client, archive):
466
response = client.get_messages('baz', max_body_length=3)
467
first_message = response['messages'][0]
468
self.assertEqual('abc', first_message['body'])
469
470
def test_include_hidden(self):
471
client = GrackleClient('localhost', 8444)
472
archive = {
473
'baz': [
474
make_message('foo', hidden=True),
475
make_message('bar', hidden=False),
476
]}
477
with ForkedFakeService.from_client(client, archive):
478
response = client.get_messages('baz', include_hidden=True)
479
self.assertMessageIDs(['bar', 'foo'], response['messages'])
480
response = client.get_messages('baz', include_hidden=False)
481
self.assertMessageIDs(['bar'], response['messages'])
482
483
def test_display_type_unknown_value(self):
484
client = GrackleClient('localhost', 8445)
485
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
486
with ForkedFakeService.from_client(client, archive):
487
with ExpectedException(UnsupportedDisplayType, ''):
488
client.get_messages('baz', display_type='unknown')
489
490
def test_display_type_headers_only(self):
491
client = GrackleClient('localhost', 8446)
492
archive = {
493
'baz': [
494
make_message('foo', body=u'abcdefghi',
495
headers={'From': 'me', 'To': 'you'})]}
496
with ForkedFakeService.from_client(client, archive):
497
response = client.get_messages('baz', display_type='headers-only')
498
first_message = response['messages'][0]
499
self.assertEqual('foo', first_message['message_id'])
500
self.assertEqual(
501
{'From': 'me', 'Message-Id': 'foo', 'To': 'you'},
502
first_message['headers'])
503
self.assertNotIn('body', first_message)
504
505
def test_display_type_text_only(self):
506
client = GrackleClient('localhost', 8446)
507
archive = {
508
'baz': [
509
make_mime_message(
510
'foo', 'abcdefghi',
511
headers={'From': 'me', 'To': 'you'},
512
attachment_type='text/x-diff')]}
513
with ForkedFakeService.from_client(client, archive):
514
response = client.get_messages('baz', display_type='text-only')
515
first_message = response['messages'][0]
516
self.assertEqual('foo', first_message['message_id'])
517
self.assertEqual('me', first_message['headers']['From'])
518
self.assertEqual('you', first_message['headers']['To'])
519
self.assertEqual('abcdefghi', first_message['body'])
520
521
def test_display_type_all(self):
522
client = GrackleClient('localhost', 8447)
523
archive = {
524
'baz': [
525
make_mime_message(
526
'foo', 'abcdefghi',
527
headers={'From': 'me', 'To': 'you'},
528
attachment_type='text/x-diff')]}
529
with ForkedFakeService.from_client(client, archive):
530
response = client.get_messages('baz', display_type='all')
531
first_message = response['messages'][0]
532
self.assertEqual('foo', first_message['message_id'])
533
self.assertEqual('me', first_message['headers']['From'])
534
self.assertEqual('you', first_message['headers']['To'])
535
self.assertEqual(
536
'abcdefghi\n\nattactment data.', first_message['body'])
Loggerhead is a web-based interface for Breezy
Version: 2.0.1