« back to all changes in this revision
Viewing changes to grackle/tests/test_client.py
- browse files at revision 38
- compare with another revision
- download diff
- download tarball
- view history from revision 38
- Committer: Curtis Hovey
- Date: 2012-02-14 21:55:15 UTC
- Revision ID: curtis.hovey@canonical.com-20120214215515-vu1zn9n58ov659es
Added basic handling of date_range.
- files modified:
- grackle-put-message
added
removed
grackle/tests/test_client.py
2
2
HTTPServer,
3
3
BaseHTTPRequestHandler,
4
4
)
5
from email.message import Message
6
from email.mime.multipart import MIMEMultipart
7
from email.mime.text import MIMEText
5
8
import httplib
9
import logging
6
10
import os
7
11
from signal import SIGKILL
12
import simplejson
8
13
from StringIO import StringIO
14
import sys
9
15
from unittest import TestCase
16
from urlparse import urlparse
17
from urlparse import parse_qs
10
18
11
19
from testtools import ExpectedException
12
20
13
21
from grackle.client import (
14
22
GrackleClient,
23
UnparsableDateRange,
24
UnsupportedDisplayType,
25
UnsupportedOrder,
15
26
)
16
27
17
28
18
class Forked:
19
20
def __init__(self, func_or_method, *args):
21
self.func_or_method = func_or_method
29
def make_message(message_id, body='body', headers=None, hidden=False):
30
if headers is None:
31
headers = {}
32
headers['Message-Id'] = message_id
33
message = {
34
'message_id': message_id,
35
'headers': headers,
36
'thread_id': message_id,
37
'date': headers.get('date', '2005-01-01'),
38
'subject': headers.get('subject', 'subject'),
39
'author': headers.get('author', 'author'),
40
'hidden': hidden,
41
'attachments': [],
42
'replies': headers.get('in-reply-to', None),
43
'body': body,
44
}
45
return message
46
47
48
def make_mime_message(message_id, body='body', headers=None, hidden=False,
49
attachment_type=None):
50
message = MIMEMultipart()
51
message.attach(MIMEText(body))
52
if attachment_type is not None:
53
attachment = Message()
54
attachment.set_payload('attactment data.')
55
attachment['Content-Type'] = attachment_type
56
attachment['Content-Disposition'] = 'attachment; filename="file.ext"'
57
message.attach(attachment)
58
return make_message(message_id, message.get_payload(), headers, hidden)
59
60
61
def threaded_messages(messages):
62
threads = {}
63
count = 0
64
pending = []
65
for message in messages:
66
if message.get('replies') is None:
67
threads[message['message_id']] = [message]
68
count += 1
69
else:
70
pending.append(message)
71
for message in pending:
72
threads[message['replies']].append(message)
73
return threads.values()
74
75
76
class GrackleStore:
77
"""A memory-backed message store."""
78
79
def __init__(self, messages):
80
"""Constructor."""
81
self.messages = messages
82
83
@staticmethod
84
def is_multipart(message):
85
return isinstance(message['body'], list)
86
87
def get_messages(self, archive_id, query_string):
88
"""Return matching messages.
89
90
:param archive_id: The archive to retrieve from.
91
:param query_string: Contains 'parameters', which is a JSON-format
92
string describing parameters.
93
"""
94
query = parse_qs(query_string)
95
parameters = simplejson.loads(query['parameters'][0])
96
order = parameters.get('order')
97
messages = self.messages[archive_id]
98
if order is not None:
99
if order not in SUPPORTED_ORDERS:
100
raise UnsupportedOrder
101
elif order.startswith('thread_'):
102
threaded = threaded_messages(messages)
103
messages = []
104
if order == 'thread_subject':
105
threaded.sort(key=lambda t: t[0]['subject'])
106
if order == 'thread_oldest':
107
threaded.sort(key=lambda t: min(m['date'] for m in t))
108
if order == 'thread_newest':
109
threaded.sort(key=lambda t: max(m['date'] for m in t))
110
for thread in threaded:
111
messages.extend(thread)
112
else:
113
messages.sort(key=lambda m: m[order])
114
display_type = parameters.get('display_type', 'all')
115
if display_type not in SUPPORTED_DISPLAY_TYPES:
116
raise UnsupportedDisplayType
117
if 'date_range' in parameters:
118
try:
119
start_date, end_date = parameters['date_range'].split('..')
120
except:
121
raise UnparsableDateRange()
122
new_messages = []
123
for message in messages:
124
if (not parameters['include_hidden'] and message['hidden']):
125
continue
126
if ('message_ids' in parameters
127
and message['message_id'] not in parameters['message_ids']):
128
continue
129
if ('date_range' in parameters
130
and (message['date'] < start_date
131
or message['date'] > end_date)):
132
continue
133
message = dict(message)
134
if 'headers' in parameters:
135
headers = dict(
136
(k, v) for k, v in message['headers'].iteritems()
137
if k in parameters['headers'])
138
message['headers'] = headers
139
if display_type == 'headers-only':
140
del message['body']
141
elif display_type == 'text-only' and self.is_multipart(message):
142
text_parts = [
143
part.get_payload() for part in message['body']
144
if part.get_content_type() == 'text/plain']
145
message['body'] = '\n\n'.join(text_parts)
146
elif display_type == 'all' and self.is_multipart(message):
147
parts = [str(part.get_payload()) for part in message['body']]
148
message['body'] = '\n\n'.join(parts)
149
max_body = parameters.get('max_body_length')
150
if max_body is not None and display_type != 'headers-only':
151
message['body'] = message['body'][:max_body]
152
new_messages.append(message)
153
messages = new_messages
154
limit = parameters.get('limit', 100)
155
memo = parameters.get('memo')
156
message_id_indices = dict(
157
(m['message_id'], idx) for idx, m in enumerate(messages))
158
if memo is None:
159
start = 0
160
else:
161
start = message_id_indices[memo.encode('rot13')]
162
if start > 0:
163
previous_memo = messages[start - 1]['message_id'].encode('rot13')
164
else:
165
previous_memo = None
166
end = min(start + limit, len(messages))
167
if end < len(messages):
168
next_memo = messages[end]['message_id'].encode('rot13')
169
else:
170
next_memo = None
171
messages = messages[start:end]
172
173
response = {
174
'messages': messages,
175
'next_memo': next_memo,
176
'previous_memo': previous_memo
177
}
178
return response
179
180
181
class ForkedFakeService:
182
"""A Grackle service fake, as a ContextManager."""
183
184
def __init__(self, port, messages=None, write_logs=False):
185
"""Constructor.
186
187
:param port: The tcp port to use.
188
:param messages: A dict of lists of dicts representing messages. The
189
outer dict represents the archive, the list represents the list of
190
messages for that archive.
191
:param write_logs: If true, log messages will be written to stdout.
192
"""
22
193
self.pid = None
23
self.args = args
194
self.port = port
195
if messages is None:
196
self.messages = {}
197
else:
198
self.messages = messages
199
self.read_end, self.write_end = os.pipe()
200
self.write_logs = write_logs
201
202
@staticmethod
203
def from_client(client, messages=None):
204
"""Instantiate a ForkedFakeService from the client.
205
206
:param port: The client to provide service for.
207
:param messages: A dict of lists of dicts representing messages. The
208
outer dict represents the archive, the list represents the list of
209
messages for that archive.
210
"""
211
return ForkedFakeService(client.port, messages)
212
213
def is_ready(self):
214
"""Tell the parent process that the server is ready for writes."""
215
os.write(self.write_end, 'asdf')
24
216
25
217
def __enter__(self):
218
"""Run the service.
219
220
Fork and start a server in the child. Return when the server is ready
221
for use."""
26
222
pid = os.fork()
27
if pid != 0:
28
self.pid = pid
29
return
30
self.func_or_method(*self.args)
223
if pid == 0:
224
self.start_server()
225
self.pid = pid
226
os.read(self.read_end, 1)
227
return
31
228
229
def start_server(self):
230
"""Start the HTTP server."""
231
service = HTTPServer(('', self.port), FakeGrackleRequestHandler)
232
service.store = GrackleStore(self.messages)
233
for archive_id, messages in service.store.messages.iteritems():
234
for message in messages:
235
message.setdefault('headers', {})
236
self.is_ready()
237
if self.write_logs:
238
logging.basicConfig(
239
stream=sys.stderr, level=logging.INFO)
240
service.serve_forever()
32
241
33
242
def __exit__(self, exc_type, exc_val, traceback):
34
243
os.kill(self.pid, SIGKILL)
35
244
36
245
246
SUPPORTED_DISPLAY_TYPES = set(['all', 'text-only', 'headers-only'])
247
248
249
SUPPORTED_ORDERS = set(
250
['date', 'author', 'subject', 'thread_newest', 'thread_oldest',
251
'thread_subject'])
252
253
37
254
class FakeGrackleRequestHandler(BaseHTTPRequestHandler):
255
"""A request handler that forwards to server.store."""
256
257
def __init__(self, *args, **kwargs):
258
"""Constructor. Sets up logging."""
259
self.logger = logging.getLogger('http')
260
BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
38
261
39
262
def do_POST(self):
263
"""Create a message on POST."""
40
264
message = self.rfile.read(int(self.headers['content-length']))
41
265
if message == 'This is a message':
42
266
self.send_response(httplib.CREATED)
45
269
else:
46
270
self.send_error(httplib.BAD_REQUEST)
47
271
48
49
def run_service(port, messages):
50
service = HTTPServer(('', port), FakeGrackleRequestHandler)
51
service.messages = messages
52
service.serve_forever()
53
54
55
def fake_grackle_service(client, messages=None):
56
if messages is None:
57
messages = {}
58
return Forked(run_service, client.port, messages)
272
def do_GET(self):
273
"""Retrieve a list of messages on GET."""
274
scheme, netloc, path, params, query_string, fragments = (
275
urlparse(self.path))
276
parts = path.split('/')
277
if parts[1] == 'archive':
278
try:
279
response = self.server.store.get_messages(
280
parts[2], query_string)
281
self.send_response(httplib.OK)
282
self.end_headers()
283
self.wfile.write(simplejson.dumps(response))
284
except UnsupportedOrder:
285
self.send_response(
286
httplib.BAD_REQUEST, UnsupportedOrder.__doc__)
287
return
288
except UnsupportedDisplayType:
289
self.send_response(
290
httplib.BAD_REQUEST, UnsupportedDisplayType.__doc__)
291
return
292
293
def log_message(self, format, *args):
294
"""Override log_message to use standard Python logging."""
295
message = "%s - - [%s] %s\n" % (
296
self.address_string(), self.log_date_time_string(), format % args)
297
self.logger.info(message)
59
298
60
299
61
300
class TestPutMessage(TestCase):
62
301
63
302
def test_put_message(self):
64
client = GrackleClient('localhost', 8435)
65
with fake_grackle_service(client):
66
import time; time.sleep(0.5)
303
client = GrackleClient('localhost', 8436)
304
with ForkedFakeService.from_client(client):
67
305
client.put_message('arch1', 'asdf', StringIO('This is a message'))
68
306
with ExpectedException(Exception, 'wtf'):
69
307
client.put_message('arch1', 'asdf',
72
310
73
311
class TestGetMessages(TestCase):
74
312
313
def assertIDOrder(self, ids, messages):
314
self.assertEqual(ids, [m['message_id'] for m in messages])
315
316
def assertMessageIDs(self, ids, messages):
317
self.assertIDOrder(
318
sorted(ids), sorted(messages, key=lambda m: m['message_id']))
319
75
320
def test_get_messages(self):
76
321
client = GrackleClient('localhost', 8435)
77
with fake_grackle_service(client,
78
{'baz':
79
[{'message-id': 'foo'},
80
{'message-id': 'bar'}]}):
81
response = client.get_messages('baz')
82
self.assertEqual(['bar', 'foo'], sorted(response.keys()))
322
archive = {
323
'baz': [make_message('foo'), make_message('bar')]}
324
with ForkedFakeService.from_client(client, archive):
325
response = client.get_messages('baz')
326
self.assertEqual(['bar', 'foo'], sorted(m['message_id'] for m in
327
response['messages']))
328
self.assertIs(None, response['next_memo'])
329
self.assertIs(None, response['previous_memo'])
330
331
def test_get_messages_by_id(self):
332
client = GrackleClient('localhost', 8437)
333
archive = {
334
'baz': [make_message('foo'), make_message('bar')]}
335
with ForkedFakeService.from_client(client, archive):
336
response = client.get_messages('baz', message_ids=['foo'])
337
message, = response['messages']
338
self.assertEqual('foo', message['message_id'])
339
340
def test_get_messages_batching(self):
341
client = GrackleClient('localhost', 8438)
342
archive = {'baz': [make_message('foo'), make_message('bar')]}
343
with ForkedFakeService.from_client(client, archive):
344
response = client.get_messages('baz', limit=1)
345
self.assertEqual(1, len(response['messages']))
346
messages = response['messages']
347
response = client.get_messages(
348
'baz', limit=1, memo=response['next_memo'])
349
self.assertEqual(1, len(response['messages']))
350
messages.extend(response['messages'])
351
self.assertMessageIDs(['foo', 'bar'], messages)
352
353
def get_messages_member_order_test(self, key):
354
client = GrackleClient('localhost', 8439)
355
archive = {
356
'baz': [
357
make_message('foo', headers={key: '2011-03-25'}),
358
make_message('bar', headers={key: '2011-03-24'}),
359
]}
360
with ForkedFakeService.from_client(client, archive):
361
response = client.get_messages('baz')
362
self.assertIDOrder(['foo', 'bar'], response['messages'])
363
response = client.get_messages('baz', order=key)
364
self.assertIDOrder(['bar', 'foo'], response['messages'])
365
366
def test_get_messages_date_order(self):
367
self.get_messages_member_order_test('date')
368
369
def test_get_messages_author_order(self):
370
self.get_messages_member_order_test('author')
371
372
def test_get_messages_subject_order(self):
373
self.get_messages_member_order_test('subject')
374
375
def test_get_messages_thread_subject_order(self):
376
archive = {
377
'baz': [
378
make_message('bar', headers={'subject': 'y'}),
379
make_message('qux', headers={'subject': 'z'}),
380
make_message('foo', headers={'subject': 'x',
381
'in-reply-to': 'qux'}),
382
]}
383
client = GrackleClient('localhost', 8439)
384
with ForkedFakeService.from_client(client, archive):
385
response = client.get_messages('baz')
386
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
387
response = client.get_messages('baz', order='subject')
388
self.assertIDOrder(['foo', 'bar', 'qux'], response['messages'])
389
response = client.get_messages('baz', order='thread_subject')
390
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
391
392
def test_get_messages_thread_oldest_order(self):
393
client = GrackleClient('localhost', 8439)
394
archive = {
395
'baz': [
396
make_message('bar', headers={'date': 'x'}),
397
make_message('qux', headers={'date': 'z'}),
398
make_message('foo', headers={'date': 'y',
399
'in-reply-to': 'qux'}),
400
]}
401
with ForkedFakeService.from_client(client, archive):
402
response = client.get_messages('baz')
403
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
404
response = client.get_messages('baz', order='date')
405
self.assertIDOrder(['bar', 'foo', 'qux'], response['messages'])
406
response = client.get_messages('baz', order='thread_oldest')
407
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
408
409
def test_get_messages_thread_newest_order(self):
410
client = GrackleClient('localhost', 8439)
411
archive = {
412
'baz': [
413
make_message('bar', headers={'date': 'x'}),
414
make_message('qux', headers={'date': 'w'}),
415
make_message('foo', headers={'date': 'y',
416
'in-reply-to': 'bar'}),
417
make_message('baz', headers={'date': 'z',
418
'in-reply-to': 'qux'}),
419
]}
420
with ForkedFakeService.from_client(client, archive):
421
response = client.get_messages('baz', order='date')
422
self.assertIDOrder(
423
['qux', 'bar', 'foo', 'baz'], response['messages'])
424
response = client.get_messages('baz', order='thread_newest')
425
self.assertIDOrder(
426
['bar', 'foo', 'qux', 'baz'], response['messages'])
427
428
def test_get_messages_unsupported_order(self):
429
client = GrackleClient('localhost', 8439)
430
archive = {
431
'baz': [
432
make_message('foo', headers={'date': '2011-03-25'}),
433
make_message('foo', headers={'date': '2011-03-24'}),
434
]}
435
with ForkedFakeService.from_client(client, archive):
436
with ExpectedException(UnsupportedOrder, ''):
437
client.get_messages('baz', order='nonsense')
438
439
def test_get_messages_headers_no_headers(self):
440
client = GrackleClient('localhost', 8440)
441
archive = {'baz': [make_message('foo')]}
442
with ForkedFakeService.from_client(client, archive):
443
response = client.get_messages('baz', headers=[
444
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
445
first_message = response['messages'][0]
446
self.assertEqual('foo', first_message['message_id'])
447
self.assertEqual({}, first_message['headers'])
448
449
def test_get_messages_headers_exclude_headers(self):
450
client = GrackleClient('localhost', 8441)
451
archive = {
452
'baz': [make_message('foo', headers={'From': 'me'})]}
453
with ForkedFakeService.from_client(client, archive):
454
response = client.get_messages('baz', headers=[
455
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
456
first_message = response['messages'][0]
457
self.assertEqual('foo', first_message['message_id'])
458
self.assertEqual({}, first_message['headers'])
459
460
def test_get_messages_headers_include_headers(self):
461
client = GrackleClient('localhost', 8442)
462
archive = {
463
'baz': [
464
make_message('foo', headers={'From': 'me', 'To': 'you'})]}
465
with ForkedFakeService.from_client(client, archive):
466
response = client.get_messages('baz', headers=[
467
'From', 'To'])
468
first_message = response['messages'][0]
469
self.assertEqual('foo', first_message['message_id'])
470
self.assertEqual({'From': 'me', 'To': 'you'}, first_message['headers'])
471
472
def test_get_messages_max_body_length(self):
473
client = GrackleClient('localhost', 8443)
474
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
475
with ForkedFakeService.from_client(client, archive):
476
response = client.get_messages('baz', max_body_length=3)
477
first_message = response['messages'][0]
478
self.assertEqual('abc', first_message['body'])
479
480
def test_include_hidden(self):
481
client = GrackleClient('localhost', 8444)
482
archive = {
483
'baz': [
484
make_message('foo', hidden=True),
485
make_message('bar', hidden=False),
486
]}
487
with ForkedFakeService.from_client(client, archive):
488
response = client.get_messages('baz', include_hidden=True)
489
self.assertMessageIDs(['bar', 'foo'], response['messages'])
490
response = client.get_messages('baz', include_hidden=False)
491
self.assertMessageIDs(['bar'], response['messages'])
492
493
def test_display_type_unknown_value(self):
494
client = GrackleClient('localhost', 8445)
495
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
496
with ForkedFakeService.from_client(client, archive):
497
with ExpectedException(UnsupportedDisplayType, ''):
498
client.get_messages('baz', display_type='unknown')
499
500
def test_display_type_headers_only(self):
501
client = GrackleClient('localhost', 8446)
502
archive = {
503
'baz': [
504
make_message('foo', body=u'abcdefghi',
505
headers={'From': 'me', 'To': 'you'})]}
506
with ForkedFakeService.from_client(client, archive):
507
response = client.get_messages('baz', display_type='headers-only')
508
first_message = response['messages'][0]
509
self.assertEqual('foo', first_message['message_id'])
510
self.assertEqual(
511
{'From': 'me', 'Message-Id': 'foo', 'To': 'you'},
512
first_message['headers'])
513
self.assertNotIn('body', first_message)
514
515
def test_display_type_text_only(self):
516
client = GrackleClient('localhost', 8446)
517
archive = {
518
'baz': [
519
make_mime_message(
520
'foo', 'abcdefghi',
521
headers={'From': 'me', 'To': 'you'},
522
attachment_type='text/x-diff')]}
523
with ForkedFakeService.from_client(client, archive):
524
response = client.get_messages('baz', display_type='text-only')
525
first_message = response['messages'][0]
526
self.assertEqual('foo', first_message['message_id'])
527
self.assertEqual('me', first_message['headers']['From'])
528
self.assertEqual('you', first_message['headers']['To'])
529
self.assertEqual('abcdefghi', first_message['body'])
530
531
def test_display_type_all(self):
532
client = GrackleClient('localhost', 8447)
533
archive = {
534
'baz': [
535
make_mime_message(
536
'foo', 'abcdefghi',
537
headers={'From': 'me', 'To': 'you'},
538
attachment_type='text/x-diff')]}
539
with ForkedFakeService.from_client(client, archive):
540
response = client.get_messages('baz', display_type='all')
541
first_message = response['messages'][0]
542
self.assertEqual('foo', first_message['message_id'])
543
self.assertEqual('me', first_message['headers']['From'])
544
self.assertEqual('you', first_message['headers']['To'])
545
self.assertEqual(
546
'abcdefghi\n\nattactment data.', first_message['body'])
547
548
def test_date_range(self):
549
client = GrackleClient('localhost', 8448)
550
archive = {
551
'baz': [
552
make_mime_message(
553
'foo', 'abcdefghi', headers={'date': '2011-12-31'}),
554
make_mime_message(
555
'bar', 'abcdefghi', headers={'date': '2012-01-01'}),
556
make_mime_message(
557
'qux', 'abcdefghi', headers={'date': '2012-01-15'}),
558
make_mime_message(
559
'naf', 'abcdefghi', headers={'date': '2012-01-31'}),
560
make_mime_message(
561
'doh', 'abcdefghi', headers={'date': '2012-02-01'}),
562
]}
563
with ForkedFakeService.from_client(client, archive):
564
response = client.get_messages(
565
'baz', date_range='2012-01-01..2012-01-31')
566
ids = sorted(m['message_id'] for m in response['messages'])
567
self.assertEqual(['bar', 'naf', 'qux'], ids)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1