~didrocks/unity/altf10 : revision 28

2

HTTPServer,

3

BaseHTTPRequestHandler,

4

)

5

from email.message import Message

6

from email.mime.multipart import MIMEMultipart

7

from email.mime.text import MIMEText

8

5

import httplib

9

import logging

10

6

import os

11

7

from signal import SIGKILL

12

8

import simplejson

13

9

from StringIO import StringIO

14

import sys

15

10

from unittest import TestCase

16

11

from urlparse import urlparse

17

12

from urlparse import parse_qs

20

15

21

16

from grackle.client import (

22

17

GrackleClient,

23

UnparsableDateRange,

24

UnsupportedDisplayType,

25

18

UnsupportedOrder,

26

19

)

27

20

28

21

29

def make_message(message_id, body='body', headers=None, hidden=False):

30

if headers is None:

31

headers = {}

32

headers['Message-Id'] = message_id

33

message = {

34

'message_id': message_id,

35

'headers': headers,

36

'thread_id': message_id,

37

'date': headers.get('date', '2005-01-01'),

38

'subject': headers.get('subject', 'subject'),

39

'author': headers.get('author', 'author'),

40

'hidden': hidden,

41

'attachments': [],

42

'replies': headers.get('in-reply-to', None),

43

'body': body,

44

}

45

return message

46

47

48

def make_mime_message(message_id, body='body', headers=None, hidden=False,

49

attachment_type=None):

50

message = MIMEMultipart()

51

message.attach(MIMEText(body))

52

if attachment_type is not None:

53

attachment = Message()

54

attachment.set_payload('attactment data.')

55

attachment['Content-Type'] = attachment_type

56

attachment['Content-Disposition'] = 'attachment; filename="file.ext"'

57

message.attach(attachment)

58

return make_message(message_id, message.get_payload(), headers, hidden)

59

60

61

22

def threaded_messages(messages):

62

23

threads = {}

63

24

count = 0

64

25

pending = []

65

26

for message in messages:

66

if message.get('replies') is None:

27

if message.get('in_reply_to') is None:

67

28

threads[message['message_id']] = [message]

68

29

count += 1

69

30

else:

70

31

pending.append(message)

71

32

for message in pending:

72

threads[message['replies']].append(message)

33

threads[message['in_reply_to']].append(message)

73

34

return threads.values()

74

35

75

36

76

37

class GrackleStore:

77

"""A memory-backed message store."""

78

38

79

39

def __init__(self, messages):

80

"""Constructor."""

81

40

self.messages = messages

82

41

83

@staticmethod

84

def is_multipart(message):

85

return isinstance(message['body'], list)

86

87

42

def get_messages(self, archive_id, query_string):

88

"""Return matching messages.

89

90

:param archive_id: The archive to retrieve from.

91

:param query_string: Contains 'parameters', which is a JSON-format

92

string describing parameters.

93

"""

94

43

query = parse_qs(query_string)

95

44

parameters = simplejson.loads(query['parameters'][0])

96

45

order = parameters.get('order')

97

46

messages = self.messages[archive_id]

98

if order is not None:

47

if order is not None :

99

48

if order not in SUPPORTED_ORDERS:

100

49

raise UnsupportedOrder

101

50

elif order.startswith('thread_'):

111

60

messages.extend(thread)

112

61

else:

113

62

messages.sort(key=lambda m: m[order])

114

display_type = parameters.get('display_type', 'all')

115

if display_type not in SUPPORTED_DISPLAY_TYPES:

116

raise UnsupportedDisplayType

117

if 'date_range' in parameters:

118

try:

119

start_date, end_date = parameters['date_range'].split('..')

120

except:

121

raise UnparsableDateRange()

122

new_messages = []

123

for message in messages:

124

if (not parameters['include_hidden'] and message['hidden']):

125

continue

126

if ('message_ids' in parameters

127

and message['message_id'] not in parameters['message_ids']):

128

continue

129

if ('date_range' in parameters

130

and (message['date'] < start_date

131

or message['date'] > end_date)):

132

continue

133

message = dict(message)

134

if 'headers' in parameters:

135

headers = dict(

136

(k, v) for k, v in message['headers'].iteritems()

137

if k in parameters['headers'])

138

message['headers'] = headers

139

if display_type == 'headers-only':

140

del message['body']

141

elif display_type == 'text-only' and self.is_multipart(message):

142

text_parts = [

143

part.get_payload() for part in message['body']

144

if part.get_content_type() == 'text/plain']

145

message['body'] = '\n\n'.join(text_parts)

146

elif display_type == 'all' and self.is_multipart(message):

147

parts = [str(part.get_payload()) for part in message['body']]

148

message['body'] = '\n\n'.join(parts)

149

max_body = parameters.get('max_body_length')

150

if max_body is not None and display_type != 'headers-only':

151

message['body'] = message['body'][:max_body]

152

new_messages.append(message)

153

messages = new_messages

63

messages = [m for m in messages

64

if 'message_ids' not in parameters or

65

m['message_id'] in parameters['message_ids']]

154

66

limit = parameters.get('limit', 100)

155

67

memo = parameters.get('memo')

156

68

message_id_indices = dict(

169

81

else:

170

82

next_memo = None

171

83

messages = messages[start:end]

172

84

new_messages = []

85

for message in messages:

86

message = dict(message)

87

if 'headers' in parameters:

88

headers = dict(

89

(k, v) for k, v in message['headers'].iteritems()

90

if k in parameters['headers'])

91

message['headers'] = headers

92

max_body = parameters.get('max_body_length')

93

if max_body is not None:

94

message['body'] = message['body'][:max_body]

95

new_messages.append(message)

173

96

response = {

174

'messages': messages,

97

'messages': new_messages,

175

98

'next_memo': next_memo,

176

99

'previous_memo': previous_memo

177

100

}

178

101

return response

179

102

180

103

181

class ForkedFakeService:

182

"""A Grackle service fake, as a ContextManager."""

183

184

def __init__(self, port, messages=None, write_logs=False):

185

"""Constructor.

186

187

:param port: The tcp port to use.

188

:param messages: A dict of lists of dicts representing messages. The

189

outer dict represents the archive, the list represents the list of

190

messages for that archive.

191

:param write_logs: If true, log messages will be written to stdout.

192

"""

104

105

class ForkedFake:

106

107

def __init__(self, port, messages=None):

193

108

self.pid = None

194

109

self.port = port

195

if messages is None:

196

self.messages = {}

197

else:

198

self.messages = messages

110

self.messages = messages

199

111

self.read_end, self.write_end = os.pipe()

200

self.write_logs = write_logs

201

202

@staticmethod

203

def from_client(client, messages=None):

204

"""Instantiate a ForkedFakeService from the client.

205

206

:param port: The client to provide service for.

207

:param messages: A dict of lists of dicts representing messages. The

208

outer dict represents the archive, the list represents the list of

209

messages for that archive.

210

"""

211

return ForkedFakeService(client.port, messages)

212

112

213

113

def is_ready(self):

214

"""Tell the parent process that the server is ready for writes."""

215

114

os.write(self.write_end, 'asdf')

216

115

217

116

def __enter__(self):

218

"""Run the service.

219

220

Fork and start a server in the child. Return when the server is ready

221

for use."""

222

117

pid = os.fork()

223

118

if pid == 0:

224

119

self.start_server()

227

122

return

228

123

229

124

def start_server(self):

230

"""Start the HTTP server."""

231

125

service = HTTPServer(('', self.port), FakeGrackleRequestHandler)

232

126

service.store = GrackleStore(self.messages)

233

127

for archive_id, messages in service.store.messages.iteritems():

234

128

for message in messages:

235

129

message.setdefault('headers', {})

236

130

self.is_ready()

237

if self.write_logs:

238

logging.basicConfig(

239

stream=sys.stderr, level=logging.INFO)

240

131

service.serve_forever()

241

132

242

133

def __exit__(self, exc_type, exc_val, traceback):

243

134

os.kill(self.pid, SIGKILL)

244

135

245

136

246

SUPPORTED_DISPLAY_TYPES = set(['all', 'text-only', 'headers-only'])

247

248

249

137

SUPPORTED_ORDERS = set(

250

138

['date', 'author', 'subject', 'thread_newest', 'thread_oldest',

251

139

'thread_subject'])

252

140

253

141

254

142

class FakeGrackleRequestHandler(BaseHTTPRequestHandler):

255

"""A request handler that forwards to server.store."""

256

257

def __init__(self, *args, **kwargs):

258

"""Constructor. Sets up logging."""

259

self.logger = logging.getLogger('http')

260

BaseHTTPRequestHandler.__init__(self, *args, **kwargs)

261

143

262

144

def do_POST(self):

263

"""Create a message on POST."""

264

145

message = self.rfile.read(int(self.headers['content-length']))

265

146

if message == 'This is a message':

266

147

self.send_response(httplib.CREATED)

270

151

self.send_error(httplib.BAD_REQUEST)

271

152

272

153

def do_GET(self):

273

"""Retrieve a list of messages on GET."""

274

154

scheme, netloc, path, params, query_string, fragments = (

275

155

urlparse(self.path))

276

156

parts = path.split('/')

282

162

self.end_headers()

283

163

self.wfile.write(simplejson.dumps(response))

284

164

except UnsupportedOrder:

285

self.send_response(

286

httplib.BAD_REQUEST, UnsupportedOrder.__doc__)

287

return

288

except UnsupportedDisplayType:

289

self.send_response(

290

httplib.BAD_REQUEST, UnsupportedDisplayType.__doc__)

291

return

292

293

def log_message(self, format, *args):

294

"""Override log_message to use standard Python logging."""

295

message = "%s - - [%s] %s\n" % (

296

self.address_string(), self.log_date_time_string(), format % args)

297

self.logger.info(message)

165

self.send_response(httplib.BAD_REQUEST)

166

self.wfile.write('Unsupported order')

167

return

168

169

170

def fake_grackle_service(client, messages=None):

171

if messages is None:

172

messages = {}

173

return ForkedFake(client.port, messages)

298

174

299

175

300

176

class TestPutMessage(TestCase):

301

177

302

178

def test_put_message(self):

303

179

client = GrackleClient('localhost', 8436)

304

with ForkedFakeService.from_client(client):

180

with fake_grackle_service(client):

305

181

client.put_message('arch1', 'asdf', StringIO('This is a message'))

306

182

with ExpectedException(Exception, 'wtf'):

307

183

client.put_message('arch1', 'asdf',

315

191

316

192

def assertMessageIDs(self, ids, messages):

317

193

self.assertIDOrder(

318

sorted(ids), sorted(messages, key=lambda m: m['message_id']))

194

sorted(ids), sorted(messages, key=lambda m:m['message_id']))

319

195

320

196

def test_get_messages(self):

321

197

client = GrackleClient('localhost', 8435)

322

archive = {

323

'baz': [make_message('foo'), make_message('bar')]}

324

with ForkedFakeService.from_client(client, archive):

198

with fake_grackle_service(client,

199

{'baz':

200

[{'message_id': 'foo'},

201

{'message_id': 'bar'}]}):

325

202

response = client.get_messages('baz')

326

203

self.assertEqual(['bar', 'foo'], sorted(m['message_id'] for m in

327

204

response['messages']))

330

207

331

208

def test_get_messages_by_id(self):

332

209

client = GrackleClient('localhost', 8437)

333

archive = {

334

'baz': [make_message('foo'), make_message('bar')]}

335

with ForkedFakeService.from_client(client, archive):

210

with fake_grackle_service(client,

211

{'baz':

212

[{'message_id': 'foo'},

213

{'message_id': 'bar'}]}):

336

214

response = client.get_messages('baz', message_ids=['foo'])

337

215

message, = response['messages']

338

216

self.assertEqual('foo', message['message_id'])

339

217

340

218

def test_get_messages_batching(self):

341

219

client = GrackleClient('localhost', 8438)

342

archive = {'baz': [make_message('foo'), make_message('bar')]}

343

with ForkedFakeService.from_client(client, archive):

220

with fake_grackle_service(client,

221

{'baz':

222

[{'message_id': 'foo'},

223

{'message_id': 'bar'}]}):

344

224

response = client.get_messages('baz', limit=1)

345

225

self.assertEqual(1, len(response['messages']))

346

226

messages = response['messages']

352

232

353

233

def get_messages_member_order_test(self, key):

354

234

client = GrackleClient('localhost', 8439)

355

archive = {

356

'baz': [

357

make_message('foo', headers={key: '2011-03-25'}),

358

make_message('bar', headers={key: '2011-03-24'}),

359

]}

360

with ForkedFakeService.from_client(client, archive):

235

with fake_grackle_service(client,

236

{'baz': [{'message_id': 'foo', key: '2011-03-25'},

237

{'message_id': 'bar', key: '2011-03-24'}]}):

361

238

response = client.get_messages('baz')

362

239

self.assertIDOrder(['foo', 'bar'], response['messages'])

363

240

response = client.get_messages('baz', order=key)

373

250

self.get_messages_member_order_test('subject')

374

251

375

252

def test_get_messages_thread_subject_order(self):

376

archive = {

377

'baz': [

378

make_message('bar', headers={'subject': 'y'}),

379

make_message('qux', headers={'subject': 'z'}),

380

make_message('foo', headers={'subject': 'x',

381

'in-reply-to': 'qux'}),

382

]}

383

253

client = GrackleClient('localhost', 8439)

384

with ForkedFakeService.from_client(client, archive):

254

with fake_grackle_service(client, {'baz': [

255

{'message_id': 'bar', 'subject': 'y'},

256

{'message_id': 'qux', 'subject': 'z'},

257

{'message_id': 'foo', 'subject': 'x', 'in_reply_to': 'qux'},

258

]}):

385

259

response = client.get_messages('baz')

386

260

self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])

387

261

response = client.get_messages('baz', order='subject')

391

265

392

266

def test_get_messages_thread_oldest_order(self):

393

267

client = GrackleClient('localhost', 8439)

394

archive = {

395

'baz': [

396

make_message('bar', headers={'date': 'x'}),

397

make_message('qux', headers={'date': 'z'}),

398

make_message('foo', headers={'date': 'y',

399

'in-reply-to': 'qux'}),

400

]}

401

with ForkedFakeService.from_client(client, archive):

268

with fake_grackle_service(client, {'baz': [

269

{'message_id': 'bar', 'date': 'x'},

270

{'message_id': 'qux', 'date': 'z'},

271

{'message_id': 'foo', 'date': 'y', 'in_reply_to': 'qux'},

272

]}):

402

273

response = client.get_messages('baz')

403

274

self.assertIDOrder(['bar', 'qux', 'foo'], response['messages'])

404

275

response = client.get_messages('baz', order='date')

408

279

409

280

def test_get_messages_thread_newest_order(self):

410

281

client = GrackleClient('localhost', 8439)

411

archive = {

412

'baz': [

413

make_message('bar', headers={'date': 'x'}),

414

make_message('qux', headers={'date': 'w'}),

415

make_message('foo', headers={'date': 'y',

416

'in-reply-to': 'bar'}),

417

make_message('baz', headers={'date': 'z',

418

'in-reply-to': 'qux'}),

419

]}

420

with ForkedFakeService.from_client(client, archive):

282

with fake_grackle_service(client, {'baz': [

283

{'message_id': 'bar', 'date': 'x'},

284

{'message_id': 'qux', 'date': 'w'},

285

{'message_id': 'foo', 'date': 'y', 'in_reply_to': 'bar'},

286

{'message_id': 'baz', 'date': 'z', 'in_reply_to': 'qux'},

287

]}):

421

288

response = client.get_messages('baz', order='date')

422

289

self.assertIDOrder(

423

290

['qux', 'bar', 'foo', 'baz'], response['messages'])

427

294

428

295

def test_get_messages_unsupported_order(self):

429

296

client = GrackleClient('localhost', 8439)

430

archive = {

431

'baz': [

432

make_message('foo', headers={'date': '2011-03-25'}),

433

make_message('foo', headers={'date': '2011-03-24'}),

434

]}

435

with ForkedFakeService.from_client(client, archive):

436

with ExpectedException(UnsupportedOrder, ''):

297

with fake_grackle_service(client,

298

{'baz': [{'message_id': 'foo', 'date': '2011-03-25'},

299

{'message_id': 'bar', 'date': '2011-03-24'}]}):

300

with ExpectedException(UnsupportedOrder):

437

301

client.get_messages('baz', order='nonsense')

438

302

439

303

def test_get_messages_headers_no_headers(self):

440

304

client = GrackleClient('localhost', 8440)

441

archive = {'baz': [make_message('foo')]}

442

with ForkedFakeService.from_client(client, archive):

305

with fake_grackle_service(client,

306

{'baz': [

307

{'message_id': 'foo'}

308

]}):

443

309

response = client.get_messages('baz', headers=[

444

310

'Subject', 'Date', 'X-Launchpad-Message-Rationale'])

445

311

first_message = response['messages'][0]

447

313

self.assertEqual({}, first_message['headers'])

448

314

449

315

def test_get_messages_headers_exclude_headers(self):

450

client = GrackleClient('localhost', 8441)

451

archive = {

452

'baz': [make_message('foo', headers={'From': 'me'})]}

453

with ForkedFakeService.from_client(client, archive):

316

client = GrackleClient('localhost', 8440)

317

with fake_grackle_service(client,

318

{'baz': [

319

{'message_id': 'foo', 'headers': {'From': 'me'}}

320

]}):

454

321

response = client.get_messages('baz', headers=[

455

322

'Subject', 'Date', 'X-Launchpad-Message-Rationale'])

456

323

first_message = response['messages'][0]

458

325

self.assertEqual({}, first_message['headers'])

459

326

460

327

def test_get_messages_headers_include_headers(self):

461

client = GrackleClient('localhost', 8442)

462

archive = {

463

'baz': [

464

make_message('foo', headers={'From': 'me', 'To': 'you'})]}

465

with ForkedFakeService.from_client(client, archive):

328

client = GrackleClient('localhost', 8440)

329

with fake_grackle_service(client,

330

{'baz': [

331

{'message_id': 'foo', 'headers': {'From': 'me', 'To': 'you'}}

332

]}):

466

333

response = client.get_messages('baz', headers=[

467

334

'From', 'To'])

468

335

first_message = response['messages'][0]

470

337

self.assertEqual({'From': 'me', 'To': 'you'}, first_message['headers'])

471

338

472

339

def test_get_messages_max_body_length(self):

473

client = GrackleClient('localhost', 8443)

474

archive = {'baz': [make_message('foo', body=u'abcdefghi')]}

475

with ForkedFakeService.from_client(client, archive):

340

client = GrackleClient('localhost', 8440)

341

with fake_grackle_service(client,

342

{'baz': [

343

{'message_id': 'foo', 'body': u'abcdefghi'}

344

]}):

476

345

response = client.get_messages('baz', max_body_length=3)

477

346

first_message = response['messages'][0]

478

347

self.assertEqual('abc', first_message['body'])

479

348

480

def test_include_hidden(self):

481

client = GrackleClient('localhost', 8444)

482

archive = {

483

'baz': [

484

make_message('foo', hidden=True),

485

make_message('bar', hidden=False),

486

]}

487

with ForkedFakeService.from_client(client, archive):

488

response = client.get_messages('baz', include_hidden=True)

489

self.assertMessageIDs(['bar', 'foo'], response['messages'])

490

response = client.get_messages('baz', include_hidden=False)

491

self.assertMessageIDs(['bar'], response['messages'])

492

493

def test_display_type_unknown_value(self):

494

client = GrackleClient('localhost', 8445)

495

archive = {'baz': [make_message('foo', body=u'abcdefghi')]}

496

with ForkedFakeService.from_client(client, archive):

497

with ExpectedException(UnsupportedDisplayType, ''):

498

client.get_messages('baz', display_type='unknown')

499

500

def test_display_type_headers_only(self):

501

client = GrackleClient('localhost', 8446)

502

archive = {

503

'baz': [

504

make_message('foo', body=u'abcdefghi',

505

headers={'From': 'me', 'To': 'you'})]}

506

with ForkedFakeService.from_client(client, archive):

507

response = client.get_messages('baz', display_type='headers-only')

508

first_message = response['messages'][0]

509

self.assertEqual('foo', first_message['message_id'])

510

self.assertEqual(

511

{'From': 'me', 'Message-Id': 'foo', 'To': 'you'},

512

first_message['headers'])

513

self.assertNotIn('body', first_message)

514

515

def test_display_type_text_only(self):

516

client = GrackleClient('localhost', 8446)

517

archive = {

518

'baz': [

519

make_mime_message(

520

'foo', 'abcdefghi',

521

headers={'From': 'me', 'To': 'you'},

522

attachment_type='text/x-diff')]}

523

with ForkedFakeService.from_client(client, archive):

524

response = client.get_messages('baz', display_type='text-only')

525

first_message = response['messages'][0]

526

self.assertEqual('foo', first_message['message_id'])

527

self.assertEqual('me', first_message['headers']['From'])

528

self.assertEqual('you', first_message['headers']['To'])

529

self.assertEqual('abcdefghi', first_message['body'])

530

531

def test_display_type_all(self):

532

client = GrackleClient('localhost', 8447)

533

archive = {

534

'baz': [

535

make_mime_message(

536

'foo', 'abcdefghi',

537

headers={'From': 'me', 'To': 'you'},

538

attachment_type='text/x-diff')]}

539

with ForkedFakeService.from_client(client, archive):

540

response = client.get_messages('baz', display_type='all')

541

first_message = response['messages'][0]

542

self.assertEqual('foo', first_message['message_id'])

543

self.assertEqual('me', first_message['headers']['From'])

544

self.assertEqual('you', first_message['headers']['To'])

545

self.assertEqual(

546

'abcdefghi\n\nattactment data.', first_message['body'])

547

548

def test_date_range(self):

549

client = GrackleClient('localhost', 8448)

550

archive = {

551

'baz': [

552

make_mime_message(

553

'foo', 'abcdefghi', headers={'date': '2011-12-31'}),

554

make_mime_message(

555

'bar', 'abcdefghi', headers={'date': '2012-01-01'}),

556

make_mime_message(

557

'qux', 'abcdefghi', headers={'date': '2012-01-15'}),

558

make_mime_message(

559

'naf', 'abcdefghi', headers={'date': '2012-01-31'}),

560

make_mime_message(

561

'doh', 'abcdefghi', headers={'date': '2012-02-01'}),

562

]}

563

with ForkedFakeService.from_client(client, archive):

564

response = client.get_messages(

565

'baz', date_range='2012-01-01..2012-01-31')

566

ids = sorted(m['message_id'] for m in response['messages'])

567

self.assertEqual(['bar', 'naf', 'qux'], ids)