« back to all changes in this revision
Viewing changes to grackle/tests/test_client.py
- browse files at revision 26
- compare with another revision
- download diff
- download tarball
- view history from revision 26
- Committer: Aaron Bentley
- Date: 2012-01-13 14:04:36 UTC
- Revision ID: aaron@canonical.com-20120113140436-vg1mmta8ub8o7vwr
Implement an archive namespace.
- files removed:
- grackle/error.py
- files modified:
- grackle-put-message
added
removed
grackle/tests/test_client.py
2
2
HTTPServer,
3
3
BaseHTTPRequestHandler,
4
4
)
5
from email.message import Message
6
from email.mime.multipart import MIMEMultipart
7
from email.mime.text import MIMEText
8
5
import httplib
9
import logging
10
6
import os
11
7
from signal import SIGKILL
12
8
import simplejson
13
9
from StringIO import StringIO
14
import sys
15
10
from unittest import TestCase
16
11
from urlparse import urlparse
12
from urlparse import parse_qs
17
13
18
14
from testtools import ExpectedException
19
15
20
16
from grackle.client import (
21
17
GrackleClient,
22
UnparsableDateRange,
23
UnsupportedDisplayType,
24
18
UnsupportedOrder,
25
19
)
26
from grackle.store import (
27
make_json_message,
28
MemoryStore,
29
)
30
31
32
def make_message(message_id, body='body', headers=None, hidden=False):
33
if headers is None:
34
headers = {}
35
headers['Message-Id'] = message_id
36
message = {
37
'message_id': message_id,
38
'headers': headers,
39
'thread_id': message_id,
40
'date': headers.get('date', '2005-01-01'),
41
'subject': headers.get('subject', 'subject'),
42
'author': headers.get('author', 'author'),
43
'hidden': hidden,
44
'attachments': [],
45
'replies': headers.get('in-reply-to', None),
46
'body': body,
47
}
48
return message
49
50
51
def make_mime_message(message_id, body='body', headers=None, hidden=False,
52
attachment_type=None):
53
if headers is None:
54
headers = {}
55
parts = MIMEMultipart()
56
parts.attach(MIMEText(body))
57
if attachment_type is not None:
58
attachment = Message()
59
attachment.set_payload('attactment data.')
60
attachment['Content-Type'] = attachment_type
61
attachment['Content-Disposition'] = 'attachment; filename="file.ext"'
62
parts.attach(attachment)
63
message = Message()
64
message.set_payload(parts.as_string())
65
for key, value in headers.items():
66
message[key] = value
67
return make_json_message(message_id, message.as_string())
68
69
70
class ForkedFakeService:
71
"""A Grackle service fake, as a ContextManager."""
72
73
def __init__(self, port, message_archives=None, write_logs=False):
74
"""Constructor.
75
76
:param port: The tcp port to use.
77
:param message_archives: A dict of lists of dicts representing
78
archives of messages. The outer dict represents the archive,
79
the list represents the list of messages for that archive.
80
:param write_logs: If true, log messages will be written to stdout.
81
"""
20
21
22
def threaded_messages(messages):
23
threads = {}
24
count = 0
25
pending = []
26
for message in messages:
27
if message.get('in_reply_to') is None:
28
threads[message['message_id']] = [message]
29
count += 1
30
else:
31
pending.append(message)
32
for message in pending:
33
threads[message['in_reply_to']].append(message)
34
return threads.values()
35
36
37
class ForkedFake:
38
39
def __init__(self, port, messages=None):
82
40
self.pid = None
83
41
self.port = port
84
if message_archives is None:
85
self.message_archives = {}
86
else:
87
self.message_archives = message_archives
42
self.messages = messages
88
43
self.read_end, self.write_end = os.pipe()
89
self.write_logs = write_logs
90
91
@staticmethod
92
def from_client(client, message_archives=None):
93
"""Instantiate a ForkedFakeService from the client.
94
95
:param port: The client to provide service for.
96
:param message_archives: A dict of lists of dicts representing
97
archives of messages. The outer dict represents the archive,
98
the list represents the list of messages for that archive.
99
"""
100
return ForkedFakeService(client.port, message_archives)
101
44
102
45
def is_ready(self):
103
"""Tell the parent process that the server is ready for writes."""
104
46
os.write(self.write_end, 'asdf')
105
47
106
48
def __enter__(self):
107
"""Run the service.
108
109
Fork and start a server in the child. Return when the server is ready
110
for use."""
111
49
pid = os.fork()
112
50
if pid == 0:
113
51
self.start_server()
116
54
return
117
55
118
56
def start_server(self):
119
"""Start the HTTP server."""
120
57
service = HTTPServer(('', self.port), FakeGrackleRequestHandler)
121
service.store = MemoryStore(self.message_archives)
122
for archive_id, messages in service.store.message_archives.iteritems():
123
for message in messages:
124
message.setdefault('headers', {})
58
service.messages = self.messages
125
59
self.is_ready()
126
if self.write_logs:
127
logging.basicConfig(
128
stream=sys.stderr, level=logging.INFO)
129
60
service.serve_forever()
130
61
131
62
def __exit__(self, exc_type, exc_val, traceback):
132
63
os.kill(self.pid, SIGKILL)
133
64
134
65
66
SUPPORTED_ORDERS = set(
67
['date', 'author', 'subject', 'thread_newest', 'thread_oldest',
68
'thread_subject'])
69
70
135
71
class FakeGrackleRequestHandler(BaseHTTPRequestHandler):
136
"""A request handler that forwards to server.store."""
137
138
def __init__(self, *args, **kwargs):
139
"""Constructor. Sets up logging."""
140
self.logger = logging.getLogger('http')
141
BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
142
72
143
73
def do_POST(self):
144
"""Create a message on POST."""
145
74
message = self.rfile.read(int(self.headers['content-length']))
146
scheme, netloc, path, params, query_string, fragments = (
147
urlparse(self.path))
148
parts = path.split('/')
149
if parts[1] == 'archive' and len(parts) == 4:
150
try:
151
# This expected path is /archive/archive_id/message_id.
152
self.server.store.put_message(parts[2], parts[3], message)
153
self.send_response(httplib.CREATED)
154
self.end_headers()
155
self.wfile.close()
156
except:
157
self.send_error(httplib.BAD_REQUEST)
75
if message == 'This is a message':
76
self.send_response(httplib.CREATED)
77
self.end_headers()
78
self.wfile.close()
79
else:
80
self.send_error(httplib.BAD_REQUEST)
158
81
159
82
def do_GET(self):
160
"""Retrieve a list of messages on GET."""
161
83
scheme, netloc, path, params, query_string, fragments = (
162
84
urlparse(self.path))
163
85
parts = path.split('/')
164
86
if parts[1] == 'archive':
165
try:
166
response = self.server.store.get_messages(
167
parts[2], query_string)
168
self.send_response(httplib.OK)
169
self.end_headers()
170
self.wfile.write(simplejson.dumps(response))
171
except Exception, error:
172
self.send_response(
173
httplib.BAD_REQUEST, error.__doc__)
87
self.get_messages(parts[2], query_string)
88
89
def get_messages(self, archive_id, query_string):
90
query = parse_qs(query_string)
91
parameters = simplejson.loads(query['parameters'][0])
92
order = parameters.get('order')
93
messages = self.server.messages[archive_id]
94
if order is not None :
95
if order not in SUPPORTED_ORDERS:
96
self.send_response(httplib.BAD_REQUEST)
97
self.wfile.write('Unsupported order')
174
98
return
175
176
def log_message(self, format, *args):
177
"""Override log_message to use standard Python logging."""
178
message = "%s - - [%s] %s\n" % (
179
self.address_string(), self.log_date_time_string(), format % args)
180
self.logger.info(message)
99
elif order.startswith('thread_'):
100
threaded = threaded_messages(messages)
101
messages = []
102
if order == 'thread_subject':
103
threaded.sort(key=lambda t: t[0]['subject'])
104
if order == 'thread_oldest':
105
threaded.sort(key=lambda t: min(m['date'] for m in t))
106
if order == 'thread_newest':
107
threaded.sort(key=lambda t: max(m['date'] for m in t))
108
for thread in threaded:
109
messages.extend(thread)
110
else:
111
messages.sort(key=lambda m: m[order])
112
messages = [m for m in messages
113
if 'message_ids' not in parameters or
114
m['message_id'] in parameters['message_ids']]
115
self.send_response(httplib.OK)
116
self.end_headers()
117
limit = parameters.get('limit', 100)
118
memo = parameters.get('memo')
119
message_id_indices = dict(
120
(m['message_id'], idx) for idx, m in enumerate(messages))
121
if memo is None:
122
start = 0
123
else:
124
start = message_id_indices[memo.encode('rot13')]
125
if start > 0:
126
previous_memo = messages[start - 1]['message_id'].encode('rot13')
127
else:
128
previous_memo = None
129
end = min(start + limit, len(messages))
130
if end < len(messages):
131
next_memo = messages[end]['message_id'].encode('rot13')
132
else:
133
next_memo = None
134
messages = messages[start:end]
135
response = {
136
'messages': messages,
137
'next_memo': next_memo,
138
'previous_memo': previous_memo
139
}
140
self.wfile.write(simplejson.dumps(response))
141
142
143
def fake_grackle_service(client, messages=None):
144
if messages is None:
145
messages = {}
146
return ForkedFake(client.port, messages)
181
147
182
148
183
149
class TestPutMessage(TestCase):
184
150
185
151
def test_put_message(self):
186
client = GrackleClient('localhost', 8420)
187
message_archives = {'arch1': []}
188
with ForkedFakeService.from_client(client, message_archives):
189
client.put_message('arch1', 'id1', StringIO('This is a message'))
190
response = client.get_messages('arch1')
191
self.assertEqual(1, len(response['messages']))
192
message = response['messages'][0]
193
self.assertEqual('id1', message['message_id'])
194
195
def test_put_message_without_archive(self):
196
client = GrackleClient('localhost', 8421)
197
message_archives = {'arch1': []}
198
with ForkedFakeService.from_client(client, message_archives):
152
client = GrackleClient('localhost', 8436)
153
with fake_grackle_service(client):
154
client.put_message('arch1', 'asdf', StringIO('This is a message'))
199
155
with ExpectedException(Exception, 'wtf'):
200
client.put_message('no-archive', 'id1', StringIO('message'))
156
client.put_message('arch1', 'asdf',
157
StringIO('This is not a message'))
201
158
202
159
203
160
class TestGetMessages(TestCase):
207
164
208
165
def assertMessageIDs(self, ids, messages):
209
166
self.assertIDOrder(
210
sorted(ids), sorted(messages, key=lambda m: m['message_id']))
167
sorted(ids), sorted(messages, key=lambda m:m['message_id']))
211
168
212
169
def test_get_messages(self):
213
client = GrackleClient('localhost', 8430)
214
archive = {
215
'baz': [make_message('foo'), make_message('bar')]}
216
with ForkedFakeService.from_client(client, archive):
170
client = GrackleClient('localhost', 8435)
171
with fake_grackle_service(client,
172
{'baz':
173
[{'message_id': 'foo'},
174
{'message_id': 'bar'}]}):
217
175
response = client.get_messages('baz')
218
176
self.assertEqual(['bar', 'foo'], sorted(m['message_id'] for m in
219
177
response['messages']))
222
180
223
181
def test_get_messages_by_id(self):
224
182
client = GrackleClient('localhost', 8437)
225
archive = {
226
'baz': [make_message('foo'), make_message('bar')]}
227
with ForkedFakeService.from_client(client, archive):
183
with fake_grackle_service(client,
184
{'baz':
185
[{'message_id': 'foo'},
186
{'message_id': 'bar'}]}):
228
187
response = client.get_messages('baz', message_ids=['foo'])
229
188
message, = response['messages']
230
189
self.assertEqual('foo', message['message_id'])
231
190
232
191
def test_get_messages_batching(self):
233
192
client = GrackleClient('localhost', 8438)
234
archive = {'baz': [make_message('foo'), make_message('bar')]}
235
with ForkedFakeService.from_client(client, archive):
193
with fake_grackle_service(client,
194
{'baz':
195
[{'message_id': 'foo'},
196
{'message_id': 'bar'}]}):
236
197
response = client.get_messages('baz', limit=1)
237
198
self.assertEqual(1, len(response['messages']))
238
199
messages = response['messages']
244
205
245
206
def get_messages_member_order_test(self, key):
246
207
client = GrackleClient('localhost', 8439)
247
archive = {
248
'baz': [
249
make_message('foo', headers={key: '2011-03-25'}),
250
make_message('bar', headers={key: '2011-03-24'}),
251
]}
252
with ForkedFakeService.from_client(client, archive):
208
with fake_grackle_service(client,
209
{'baz': [{'message_id': 'foo', key: '2011-03-25'},
210
{'message_id': 'bar', key: '2011-03-24'}]}):
253
211
response = client.get_messages('baz')
254
212
self.assertIDOrder(['foo', 'bar'], response['messages'])
255
213
response = client.get_messages('baz', order=key)
265
223
self.get_messages_member_order_test('subject')
266
224
267
225
def test_get_messages_thread_subject_order(self):
268
archive = {
269
'baz': [
270
make_message('bar', headers={'subject': 'y'}),
271
make_message('qux', headers={'subject': 'z'}),
272
make_message('foo', headers={'subject': 'x',
273
'in-reply-to': 'qux'}),
274
]}
275
226
client = GrackleClient('localhost', 8439)
276
with ForkedFakeService.from_client(client, archive):
227
with fake_grackle_service(client, {'baz': [
228
{'message_id': 'bar', 'subject': 'y'},
229
{'message_id': 'qux', 'subject': 'z'},
230
{'message_id': 'foo', 'subject': 'x', 'in_reply_to': 'qux'},
231
]}):
277
232
response = client.get_messages('baz')
278
233
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
279
234
response = client.get_messages('baz', order='subject')
283
238
284
239
def test_get_messages_thread_oldest_order(self):
285
240
client = GrackleClient('localhost', 8439)
286
archive = {
287
'baz': [
288
make_message('bar', headers={'date': 'x'}),
289
make_message('qux', headers={'date': 'z'}),
290
make_message('foo', headers={'date': 'y',
291
'in-reply-to': 'qux'}),
292
]}
293
with ForkedFakeService.from_client(client, archive):
241
with fake_grackle_service(client, {'baz': [
242
{'message_id': 'bar', 'date': 'x'},
243
{'message_id': 'qux', 'date': 'z'},
244
{'message_id': 'foo', 'date': 'y', 'in_reply_to': 'qux'},
245
]}):
294
246
response = client.get_messages('baz')
295
247
self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])
296
248
response = client.get_messages('baz', order='date')
300
252
301
253
def test_get_messages_thread_newest_order(self):
302
254
client = GrackleClient('localhost', 8439)
303
archive = {
304
'baz': [
305
make_message('bar', headers={'date': 'x'}),
306
make_message('qux', headers={'date': 'w'}),
307
make_message('foo', headers={'date': 'y',
308
'in-reply-to': 'bar'}),
309
make_message('baz', headers={'date': 'z',
310
'in-reply-to': 'qux'}),
311
]}
312
with ForkedFakeService.from_client(client, archive):
255
with fake_grackle_service(client, {'baz': [
256
{'message_id': 'bar', 'date': 'x'},
257
{'message_id': 'qux', 'date': 'w'},
258
{'message_id': 'foo', 'date': 'y', 'in_reply_to': 'bar'},
259
{'message_id': 'baz', 'date': 'z', 'in_reply_to': 'qux'},
260
]}):
313
261
response = client.get_messages('baz', order='date')
314
262
self.assertIDOrder(
315
263
['qux', 'bar', 'foo', 'baz'], response['messages'])
319
267
320
268
def test_get_messages_unsupported_order(self):
321
269
client = GrackleClient('localhost', 8439)
322
archive = {
323
'baz': [
324
make_message('foo', headers={'date': '2011-03-25'}),
325
make_message('foo', headers={'date': '2011-03-24'}),
326
]}
327
with ForkedFakeService.from_client(client, archive):
328
with ExpectedException(UnsupportedOrder, ''):
270
with fake_grackle_service(client,
271
{'baz': [{'message_id': 'foo', 'date': '2011-03-25'},
272
{'message_id': 'bar', 'date': '2011-03-24'}]}):
273
with ExpectedException(UnsupportedOrder):
329
274
client.get_messages('baz', order='nonsense')
330
331
def test_get_messages_headers_no_headers(self):
332
client = GrackleClient('localhost', 8440)
333
archive = {'baz': [make_message('foo')]}
334
with ForkedFakeService.from_client(client, archive):
335
response = client.get_messages('baz', headers=[
336
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
337
first_message = response['messages'][0]
338
self.assertEqual('foo', first_message['message_id'])
339
self.assertEqual({}, first_message['headers'])
340
341
def test_get_messages_headers_exclude_headers(self):
342
client = GrackleClient('localhost', 8441)
343
archive = {
344
'baz': [make_message('foo', headers={'From': 'me'})]}
345
with ForkedFakeService.from_client(client, archive):
346
response = client.get_messages('baz', headers=[
347
'Subject', 'Date', 'X-Launchpad-Message-Rationale'])
348
first_message = response['messages'][0]
349
self.assertEqual('foo', first_message['message_id'])
350
self.assertEqual({}, first_message['headers'])
351
352
def test_get_messages_headers_include_headers(self):
353
client = GrackleClient('localhost', 8442)
354
archive = {
355
'baz': [
356
make_message('foo', headers={'From': 'me', 'To': 'you'})]}
357
with ForkedFakeService.from_client(client, archive):
358
response = client.get_messages('baz', headers=[
359
'From', 'To'])
360
first_message = response['messages'][0]
361
self.assertEqual('foo', first_message['message_id'])
362
self.assertEqual({'From': 'me', 'To': 'you'}, first_message['headers'])
363
364
def test_get_messages_max_body_length(self):
365
client = GrackleClient('localhost', 8443)
366
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
367
with ForkedFakeService.from_client(client, archive):
368
response = client.get_messages('baz', max_body_length=3)
369
first_message = response['messages'][0]
370
self.assertEqual('abc', first_message['body'])
371
372
def test_include_hidden(self):
373
client = GrackleClient('localhost', 8444)
374
archive = {
375
'baz': [
376
make_message('foo', hidden=True),
377
make_message('bar', hidden=False),
378
]}
379
with ForkedFakeService.from_client(client, archive):
380
response = client.get_messages('baz', include_hidden=True)
381
self.assertMessageIDs(['bar', 'foo'], response['messages'])
382
response = client.get_messages('baz', include_hidden=False)
383
self.assertMessageIDs(['bar'], response['messages'])
384
385
def test_display_type_unknown_value(self):
386
client = GrackleClient('localhost', 8445)
387
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
388
with ForkedFakeService.from_client(client, archive):
389
with ExpectedException(UnsupportedDisplayType, ''):
390
client.get_messages('baz', display_type='unknown')
391
392
def test_display_type_headers_only(self):
393
client = GrackleClient('localhost', 8446)
394
archive = {
395
'baz': [
396
make_message('foo', body=u'abcdefghi',
397
headers={'From': 'me', 'To': 'you'})]}
398
with ForkedFakeService.from_client(client, archive):
399
response = client.get_messages('baz', display_type='headers-only')
400
first_message = response['messages'][0]
401
self.assertEqual('foo', first_message['message_id'])
402
self.assertEqual(
403
{'From': 'me', 'Message-Id': 'foo', 'To': 'you'},
404
first_message['headers'])
405
self.assertNotIn('body', first_message)
406
407
def test_display_type_text_only(self):
408
client = GrackleClient('localhost', 8446)
409
archive = {
410
'baz': [
411
make_mime_message(
412
'foo', 'abcdefghi',
413
headers={'From': 'me', 'To': 'you'},
414
attachment_type='text/x-diff')]}
415
with ForkedFakeService.from_client(client, archive):
416
response = client.get_messages('baz', display_type='text-only')
417
first_message = response['messages'][0]
418
self.assertEqual('foo', first_message['message_id'])
419
self.assertEqual('me', first_message['headers']['From'])
420
self.assertEqual('you', first_message['headers']['To'])
421
self.assertEqual(archive['baz'][0]['body'], first_message['body'])
422
423
def test_display_type_all(self):
424
client = GrackleClient('localhost', 8447)
425
archive = {
426
'baz': [
427
make_mime_message(
428
'foo', 'abcdefghi',
429
headers={'From': 'me', 'To': 'you'},
430
attachment_type='text/x-diff')]}
431
with ForkedFakeService.from_client(client, archive):
432
response = client.get_messages('baz', display_type='all')
433
first_message = response['messages'][0]
434
self.assertEqual('foo', first_message['message_id'])
435
self.assertEqual('me', first_message['headers']['From'])
436
self.assertEqual('you', first_message['headers']['To'])
437
self.assertEqual(archive['baz'][0]['body'], first_message['body'])
438
439
def test_date_range(self):
440
client = GrackleClient('localhost', 8448)
441
archive = {
442
'baz': [
443
make_mime_message(
444
'foo', 'abcdefghi', headers={'date': '2011-12-31'}),
445
make_mime_message(
446
'bar', 'abcdefghi', headers={'date': '2012-01-01'}),
447
make_mime_message(
448
'qux', 'abcdefghi', headers={'date': '2012-01-15'}),
449
make_mime_message(
450
'naf', 'abcdefghi', headers={'date': '2012-01-31'}),
451
make_mime_message(
452
'doh', 'abcdefghi', headers={'date': '2012-02-01'}),
453
]}
454
with ForkedFakeService.from_client(client, archive):
455
response = client.get_messages(
456
'baz', date_range='2012-01-01..2012-01-31')
457
ids = sorted(m['message_id'] for m in response['messages'])
458
self.assertEqual(['bar', 'naf', 'qux'], ids)
459
460
def test_date_range_unparsabledaterange(self):
461
client = GrackleClient('localhost', 8449)
462
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
463
with ForkedFakeService.from_client(client, archive):
464
with ExpectedException(UnparsableDateRange, ''):
465
client.get_messages('baz', date_range='2012-01-01')
466
467
def test_date_range_unparsabledaterange_missing_part(self):
468
client = GrackleClient('localhost', 8450)
469
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
470
with ForkedFakeService.from_client(client, archive):
471
with ExpectedException(UnparsableDateRange, ''):
472
client.get_messages('baz', date_range='2012-01-01..')
473
474
def test_date_range_unparsabledaterange_extra_part(self):
475
client = GrackleClient('localhost', 8451)
476
archive = {'baz': [make_message('foo', body=u'abcdefghi')]}
477
with ForkedFakeService.from_client(client, archive):
478
with ExpectedException(UnparsableDateRange, ''):
479
client.get_messages('baz', date_range='2012-01..12-02..12-03')
Loggerhead is a web-based interface for Breezy
Version: 2.0.1