~launchpad-pqm/launchpad/devel : revision 9265.1.1

1

#!/usr/bin/env python

2

# Run tests in a daemon.

3

#

4

5

# GNU Affero General Public License version 3 (see the file LICENSE).

6

7

__metatype__ = type

8

9

import datetime

10

import optparse

11

import os

12

import pickle

13

import re

14

import shutil

15

import subprocess

16

import sys

17

import textwrap

18

import time

19

import traceback

20

21

import bzrlib.branch

22

import bzrlib.config

23

import bzrlib.email_message

24

import bzrlib.errors

25

import bzrlib.smtp_connection

26

import bzrlib.workingtree

27

28

29

class BaseTestRunner:

30

31

def __init__(self, email=None, pqm_message=None, public_branch=None,

32

public_branch_revno=None, test_options=None):

33

self.email = email

34

self.pqm_message = pqm_message

35

self.public_branch = public_branch

36

self.public_branch_revno = public_branch_revno

37

38

# Set up the user-supplied and default testrunner options.

39

if isinstance(test_options, basestring):

40

test_options = test_options.split()

41

elif test_options is None:

42

test_options = ['-vv']

43

else:

44

# Use whatever the user passed in.

45

pass

46

self.test_options = test_options

47

48

# Configure paths.

49

self.lp_dir = os.path.join(os.path.sep, 'var', 'launchpad')

50

self.tmp_dir = os.path.join(self.lp_dir, 'tmp')

51

self.test_dir = os.path.join(self.lp_dir, 'test')

52

self.sourcecode_dir = os.path.join(self.test_dir, 'sourcecode')

53

54

# Set up logging.

55

self.logger = WebTestLogger(

56

self.test_dir,

57

self.public_branch,

58

self.public_branch_revno,

59

self.sourcecode_dir

60

)

61

62

# Daemonization options.

63

self.pid_filename = os.path.join(self.lp_dir, 'ec2test-remote.pid')

64

self.daemonized = False

65

66

def daemonize(self):

67

"""Turn the testrunner into a forked daemon process."""

68

# This also writes our pidfile to disk to a specific location. The

69

# ec2test.py --postmortem command will look there to find our PID,

70

# in order to control this process.

71

daemonize(self.pid_filename)

72

self.daemonized = True

73

74

def remove_pidfile(self):

75

if os.path.exists(self.pid_filename):

76

os.remove(self.pid_filename)

77

78

def ignore_line(self, line):

79

"""Return True if the line should be excluded from the summary log.

80

81

Defaults to False.

82

"""

83

return False

84

85

def build_test_command(self):

86

"""Return the command that will execute the test suite.

87

88

Should return a list of command options suitable for submission to

89

subprocess.call()

90

91

Subclasses must provide their own implementation of this method.

92

"""

93

raise NotImplementedError

94

95

def test(self):

96

"""Run the tests, log the results.

97

98

Signals the ec2test process and cleans up the logs once all the tests

99

have completed. If necessary, submits the branch to PQM, and mails

100

the user the test results.

101

"""

102

# We need to open the log files here because earlier calls to

103

# os.fork() may have tried to close them.

104

self.logger.prepare()

105

106

out_file = self.logger.out_file

107

summary_file = self.logger.summary_file

108

config = bzrlib.config.GlobalConfig()

109

110

call = self.build_test_command()

111

112

try:

113

try:

114

try:

115

popen = subprocess.Popen(

116

call, bufsize=-1,

117

stdout=subprocess.PIPE, stderr=subprocess.STDOUT,

118

cwd=self.test_dir)

119

120

self._gather_test_output(popen, summary_file, out_file)

121

122

# Grab the testrunner exit status

123

result = popen.wait()

124

125

if self.pqm_message is not None:

126

subject = self.pqm_message.get('Subject')

127

if result:

128

# failure

129

summary_file.write(

130

'\n\n**NOT** submitted to PQM:\n%s\n' %

131

(subject,))

132

else:

133

# success

134

conn = bzrlib.smtp_connection.SMTPConnection(

135

config)

136

conn.send_email(self.pqm_message)

137

summary_file.write('\n\nSUBMITTED TO PQM:\n%s\n' %

138

(subject,))

139

except:

140

summary_file.write('\n\nERROR IN TESTRUNNER\n\n')

141

traceback.print_exc(file=summary_file)

142

result = 1

143

raise

144

finally:

145

# It probably isn't safe to close the log files ourselves,

146

# since someone else might try to write to them later.

147

summary_file.close()

148

if self.email is not None:

149

subject = 'Test results: %s' % (result and 'FAILURE' or 'SUCCESS')

150

summary_file = open(self.logger.summary_filename, 'r')

151

bzrlib.email_message.EmailMessage.send(

152

config, self.email[0], self.email,

153

subject, summary_file.read())

154

summary_file.close()

155

finally:

156

# we do this at the end because this is a trigger to ec2test.py

157

# back at home that it is OK to kill the process and take control

158

# itself, if it wants to.

159

out_file.close()

160

self.logger.close_logs()

161

162

def _gather_test_output(self, test_process, summary_file, out_file):

163

"""Write the testrunner output to the logs."""

164

# Only write to stdout if we are running as the foreground process.

165

echo_to_stdout = not self.daemonized

166

167

last_line = ''

168

while 1:

169

data = test_process.stdout.read(256)

170

if data:

171

out_file.write(data)

172

out_file.flush()

173

if echo_to_stdout:

174

sys.stdout.write(data)

175

sys.stdout.flush()

176

lines = data.split('\n')

177

lines[0] = last_line + lines[0]

178

last_line = lines.pop()

179

for line in lines:

180

if not self.ignore_line(line):

181

summary_file.write(line + '\n')

182

summary_file.flush()

183

else:

184

summary_file.write(last_line)

185

break

186

187

188

class TestOnMergeRunner(BaseTestRunner):

189

"""Executes the Launchpad test_on_merge.py test suite."""

190

191

def build_test_command(self):

192

"""See BaseTestRunner.build_test_command()."""

193

command = ['bin/py', '-t', 'test_on_merge.py']

194

command.extend(self.test_options)

195

return command

196

197

# Used to filter lines in the summary log. See

198

# `BaseTestRunner.ignore_line()`.

199

ignore_line = re.compile(

200

r'( [\w\.\/\-]+( ?$[\w\.\/\-]+$)?|'

201

r'\s*Running.*|'

202

r'\d{4}\-\d{2}\-\d{2} \d{2}\:\d{2}\:\d{2} INFO.+|'

203

r'\s*Set up .+|'

204

r'\s*Tear down .*|'

205

r' Ran \d+ tests with .+)$').match

206

207

208

class JSCheckTestRunner(BaseTestRunner):

209

"""Executes the Launchpad JavaScript integration test suite."""

210

211

def build_test_command(self):

212

"""See BaseTestRunner.build_test_command()."""

213

# We use the xvfb server's convenience script, xvfb-run, to

214

# automagically set the display, start the command, shut down the

215

# display, and return the exit code. (See the xvfb-run man page for

216

# details.)

217

return [

218

'xvfb-run',

219

'-s', '-screen 0 1024x768x24',

220

'make', 'jscheck']

221

222

223

class WebTestLogger:

224

"""Logs test output to disk and a simple web page."""

225

226

def __init__(self, test_dir, public_branch, public_branch_revno,

227

sourcecode_dir):

228

""" Class initialiser """

229

self.test_dir = test_dir

230

self.public_branch = public_branch

231

self.public_branch_revno = public_branch_revno

232

self.sourcecode_dir = sourcecode_dir

233

234

self.www_dir = os.path.join(os.path.sep, 'var', 'www')

235

self.out_filename = os.path.join(self.www_dir, 'current_test.log')

236

self.summary_filename = os.path.join(self.www_dir, 'summary.log')

237

self.index_filename = os.path.join(self.www_dir, 'index.html')

238

239

# We will set up the preliminary bits of the web-accessible log

240

# files. "out" holds all stdout and stderr; "summary" holds filtered

241

# output; and "index" holds an index page.

242

self.out_file = None

243

self.summary_file = None

244

self.index_file = None

245

246

def open_logs(self):

247

"""Open all of our log files for writing."""

248

self.out_file = open(self.out_filename, 'w')

249

self.summary_file = open(self.summary_filename, 'w')

250

self.index_file = open(self.index_filename, 'w')

251

252

def flush_logs(self):

253

"""Flush all of our log file buffers."""

254

self.out_file.flush()

255

self.summary_file.flush()

256

self.index_file.flush()

257

258

def close_logs(self):

259

"""Closes all of the open log file handles."""

260

self.out_file.close()

261

self.summary_file.close()

262

self.index_file.close()

263

264

def prepare(self):

265

"""Prepares the log files on disk.

266

267

Writes three log files: the raw output log, the filtered "summary"

268

log file, and a HTML index page summarizing the test run paramters.

269

"""

270

self.open_logs()

271

272

out_file = self.out_file

273

summary_file = self.summary_file

274

index_file = self.index_file

275

276

def write(msg):

277

msg += '\n'

278

summary_file.write(msg)

279

out_file.write(msg)

280

msg = 'Tests started at approximately %(now)s UTC' % {

281

'now': datetime.datetime.utcnow().strftime(

282

'%a, %d %b %Y %H:%M:%S')}

283

index_file.write(textwrap.dedent('''\

284

<html>

285

<head>

286

<title>Testing</title>

287

</head>

288

<body>

289

<h1>Testing</h1>

290

291

<ul>

292

<li><a href="summary.log">Summary results</a></li>

293

<li><a href="current_test.log">Full results</a></li>

294

</ul>

295

''' % (msg,)))

296

write(msg)

297

298

index_file.write(textwrap.dedent('''\

299

<h2>Branches Tested</h2>

300

'''))

301

302

# Describe the trunk branch.

303

branch = bzrlib.branch.Branch.open_containing(self.test_dir)[0]

304

msg = '%(trunk)s, revision %(trunk_revno)d\n' % {

305

'trunk': branch.get_parent().encode('utf-8'),

306

'trunk_revno': branch.revno()}

307

index_file.write(textwrap.dedent('''\

308

309

''' % (msg,)))

310

write(msg)

311

tree = bzrlib.workingtree.WorkingTree.open(self.test_dir)

312

parent_ids = tree.get_parent_ids()

313

314

# Describe the merged branch.

315

if len(parent_ids) == 1:

316

index_file.write('<p>(no merged branch)</p>\n')

317

write('(no merged branch)')

318

else:

319

summary = branch.repository.get_revision(parent_ids[1]).get_summary()

320

data = {'name': self.public_branch.encode('utf-8'),

321

'revno': self.public_branch_revno,

322

'commit': summary}

323

324

msg = '%(name)s, revision %(revno)d (commit message: %(commit)s)\n' % data

325

index_file.write(textwrap.dedent('''\

326

<p>Merged with<br />%(msg)s</p>

327

''' % {'msg': msg}))

328

write("Merged with")

329

write(msg)

330

331

index_file.write('<dl>\n')

332

write('\nDEPENDENCY BRANCHES USED\n')

333

for name in os.listdir(self.sourcecode_dir):

334

path = os.path.join(self.sourcecode_dir, name)

335

if os.path.isdir(path):

336

try:

337

branch = bzrlib.branch.Branch.open_containing(path)[0]

338

except bzrlib.errors.NotBranchError:

339

continue

340

data = {'name': name,

341

'branch': branch.get_parent(),

342

'revno': branch.revno()}

343

write(

344

'- %(name)s\n %(branch)s\n %(revno)d\n' % data)

345

index_file.write(textwrap.dedent('''\

346

347

<dd>%(branch)s</dd>

348

<dd>%(revno)s</dd>

349

''' % data))

350

index_file.write(textwrap.dedent('''\

351

</dl>

352

</body>

353

</html>'''))

354

write('\n\nTEST RESULTS FOLLOW\n\n')

355

self.flush_logs()

356

357

358

def daemonize(pid_filename):

359

# this seems like the sort of thing that ought to be in the

360

# standard library :-/

361

pid = os.fork()

362

if (pid == 0): # child 1

363

os.setsid()

364

pid = os.fork()

365

if (pid == 0): # child 2

366

pass # lookie, we're ready to do work in the daemon

367

else:

368

os._exit(0)

369

else:

370

# give the pidfile a chance to be written before we exit.

371

time.sleep(1)

372

os._exit(0)

373

374

# write a pidfile ASAP

375

write_pidfile(pid_filename)

376

377

# Iterate through and close all file descriptors.

378

import resource

379

maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]

380

assert maxfd != resource.RLIM_INFINITY

381

for fd in range(0, maxfd):

382

try:

383

os.close(fd)

384

except OSError:

385

# we assume fd was closed

386

pass

387

os.open(os.devnull, os.O_RDWR) # this will be 0

388

os.dup2(0, 1)

389

os.dup2(0, 2)

390

391

392

def write_pidfile(pid_filename):

393

"""Write a pidfile for the current process."""

394

pid_file = open(pid_filename, "w")

395

pid_file.write(str(os.getpid()))

396

pid_file.close()

397

398

399

if __name__ == '__main__':

400

parser = optparse.OptionParser(

401

usage="%prog [options] [-- test options]",

402

description=("Build and run tests for an instance."))

403

parser.add_option(

404

'-e', '--email', action='append', dest='email', default=None,

405

help=('Email address to which results should be mailed. Defaults to '

406

'the email address from `bzr whoami`. May be supplied multiple '

407

'times. The first supplied email address will be used as the '

408

'From: address.'))

409

parser.add_option(

410

'-s', '--submit-pqm-message', dest='pqm_message', default=None,

411

help=('A base64-encoded pickle (string) of a pqm message '

412

'(bzrib.plugins.pqm.pqm_submit.PQMEmailMessage) to submit if '

413

'the test run is successful.'))

414

parser.add_option(

415

'--daemon', dest='daemon', default=False,

416

action='store_true', help=('Run test in background as daemon.'))

417

parser.add_option(

418

'--debug', dest='debug', default=False,

419

action='store_true',

420

help=('Drop to pdb trace as soon as possible.'))

421

parser.add_option(

422

'--shutdown', dest='shutdown', default=False,

423

action='store_true',

424

help=('Terminate (shutdown) instance after completion.'))

425

parser.add_option(

426

'--public-branch', dest='public_branch', default=None,

427

help=('The URL of the public branch being tested.'))

428

parser.add_option(

429

'--public-branch-revno', dest='public_branch_revno',

430

type="int", default=None,

431

help=('The revision number of the public branch being tested.'))

432

parser.add_option(

433

'--jscheck', dest='jscheck', default=False, action='store_true',

434

help=('Run the JavaScript integration test suite.'))

435

436

options, args = parser.parse_args()

437

438

if options.debug:

439

import pdb; pdb.set_trace()

440

if options.pqm_message is not None:

441

pqm_message = pickle.loads(

442

options.pqm_message.decode('string-escape').decode('base64'))

443

else:

444

pqm_message = None

445

446

if options.jscheck:

447

runner_type = JSCheckTestRunner

448

else:

449

# Use the default testrunner.

450

runner_type = TestOnMergeRunner

451

452

runner = runner_type(

453

options.email,

454

pqm_message,

455

options.public_branch,

456

options.public_branch_revno,

457

args

458

)

459

460

try:

461

try:

462

if options.daemon:

463

print 'Starting testrunner daemon...'

464

runner.daemonize()

465

466

runner.test()

467

except:

468

# Handle exceptions thrown by the test() or daemonize() methods.

469

if options.email:

470

bzrlib.email_message.EmailMessage.send(

471

bzrlib.config.GlobalConfig(), options.email[0],

472

options.email,

473

'Test Runner FAILED', traceback.format_exc())

474

raise

475

finally:

476

477

# When everything is over, if we've been ask to shut down, then

478

# make sure we're daemonized, then shutdown. Otherwise, if we're

479

# daemonized, just clean up the pidfile.

480

if options.shutdown:

481

# Make sure our process is daemonized, and has therefore

482

# disconnected the controlling terminal. This also disconnects

483

# the ec2test.py SSH connection, thus signalling ec2test.py

484

# that it may now try to take control of the server.

485

if not runner.daemonized:

486

# We only want to do this if we haven't already been

487

# daemonized. Nesting daemons is bad.

488

runner.daemonize()

489

490

# Give the script 60 seconds to clean itself up, and 60 seconds

491

# for the ec2test.py --postmortem option to grab control if

492

# needed. If we don't give --postmortem enough time to log

493

# in via SSH and take control, then this server will begin to

494

# shutdown on it's own.

495

#

496

# (FWIW, "grab control" means sending SIGTERM to this script's

497

# process id, thus preventing fail-safe shutdown.)

498

time.sleep(60)

499

500

# We'll only get here if --postmortem didn't kill us. This is

501

# our fail-safe shutdown, in case the user got disconnected

502

# or suffered some other mishap that would prevent them from

503

# shutting down this server on their own.

504

subprocess.call(['sudo', 'shutdown', '-P', 'now'])

505

elif runner.daemonized:

506

# It would be nice to clean up after ourselves, since we won't

507

# be shutting down.

508

runner.remove_pidfile()

509

else:

510

# We're not a daemon, and we're not shutting down. The user most

511

# likely started this script manually, from a shell running on the

512

# instance itself.

513

pass