~launchpad-pqm/launchpad/devel : revision 9265.1.1

1

#!/usr/bin/python

2

# Run tests on a branch in an EC2 instance.

3

#

4

5

# GNU Affero General Public License version 3 (see the file LICENSE).

6

7

__metatype__ = type

8

9

import cStringIO

10

import code

11

import optparse

12

import os

13

import pickle

14

import re

15

import select

16

import socket

17

import subprocess

18

import sys

19

import time

20

import urllib

21

import traceback

22

# The rlcompleter and readline modules change the behavior of the python

23

# interactive interpreter just by being imported.

24

import readline

25

import rlcompleter

26

# Shut up pyflakes.

27

rlcompleter

28

29

import boto

30

from boto.exception import EC2ResponseError

31

from bzrlib.branch import Branch

32

from bzrlib.bzrdir import BzrDir

33

from bzrlib.config import GlobalConfig

34

from bzrlib.errors import UncommittedChanges

35

from bzrlib.plugins.launchpad.account import get_lp_login

36

from bzrlib.plugins.pqm.pqm_submit import (

37

NoPQMSubmissionAddress, PQMSubmission)

38

import paramiko

39

40

41

TRUNK_BRANCH = 'bzr+ssh://bazaar.launchpad.net/~launchpad-pqm/launchpad/devel'

42

DEFAULT_INSTANCE_TYPE = 'c1.xlarge'

43

AVAILABLE_INSTANCE_TYPES = ('m1.large', 'm1.xlarge', 'c1.xlarge')

44

VALID_AMI_OWNERS = (

45

255383312499, # gary

46

559320013529, # flacoste

47

200337130613, # mwhudson

48

# ...anyone else want in on the fun?

49

)

50

51

readline.parse_and_bind('tab: complete')

52

53

#############################################################################

54

# Try to guide users past support problems we've encountered before

55

if boto.Version != '1.5b':

56

msg = [

57

'Your version of python-boto (%s) is not supported.' %(boto.Version,),

58

'See https://wiki.canonical.com/Launchpad/Experiments/FiveMinutesPQM '

59

'for instructions, or search for "python-boto" among the PPAs at '

60

'https://launchpad.net/~launchpad/+archive/ppa.']

61

if not re.match('/var/lib/python-support/python2.[56]/boto',

62

os.path.dirname(boto.__file__)):

63

# Was this boto supplied by a deb?

64

msg.append('WARNING: it looks like your version of python-boto '

65

'might not be from a debian package. It may be masking '

66

'the version installed by the package manager.')

67

raise RuntimeError(' '.join(msg))

68

if not paramiko.__version__.startswith('1.7.4'):

69

raise RuntimeError('Your version of paramiko (%s) is not supported. '

70

'Please use 1.7.4.' % (paramiko.__version__,))

71

# maybe add similar check for bzrlib?

72

# End

73

#############################################################################

74

75

#############################################################################

76

# Modified from paramiko.config. The change should be pushed upstream.

77

# Our fork supports Host lines with more than one host.

78

79

import fnmatch

80

81

82

class SSHConfig (object):

83

"""

84

Representation of config information as stored in the format used by

85

OpenSSH. Queries can be made via L{lookup}. The format is described in

86

OpenSSH's C{ssh_config} man page. This class is provided primarily as a

87

convenience to posix users (since the OpenSSH format is a de-facto

88

standard on posix) but should work fine on Windows too.

89

90

@since: 1.6

91

"""

92

93

def __init__(self):

94

"""

95

Create a new OpenSSH config object.

96

"""

97

self._config = [ { 'host': '*' } ]

98

99

def parse(self, file_obj):

100

"""

101

Read an OpenSSH config from the given file object.

102

103

@param file_obj: a file-like object to read the config file from

104

@type file_obj: file

105

"""

106

configs = [self._config[0]]

107

for line in file_obj:

108

line = line.rstrip('\n').lstrip()

109

if (line == '') or (line[0] == '#'):

110

continue

111

if '=' in line:

112

key, value = line.split('=', 1)

113

key = key.strip().lower()

114

else:

115

# find first whitespace, and split there

116

i = 0

117

while (i < len(line)) and not line[i].isspace():

118

i += 1

119

if i == len(line):

120

raise Exception('Unparsable line: %r' % line)

121

key = line[:i].lower()

122

value = line[i:].lstrip()

123

124

if key == 'host':

125

del configs[:]

126

# the value may be multiple hosts, space-delimited

127

for host in value.split():

128

# do we have a pre-existing host config to append to?

129

matches = [c for c in self._config if c['host'] == host]

130

if len(matches) > 0:

131

configs.append(matches[0])

132

else:

133

config = { 'host': host }

134

self._config.append(config)

135

configs.append(config)

136

else:

137

for config in configs:

138

config[key] = value

139

140

def lookup(self, hostname):

141

"""

142

Return a dict of config options for a given hostname.

143

144

The host-matching rules of OpenSSH's C{ssh_config} man page are used,

145

which means that all configuration options from matching host

146

specifications are merged, with more specific hostmasks taking

147

precedence. In other words, if C{"Port"} is set under C{"Host *"}

148

and also C{"Host *.example.com"}, and the lookup is for

149

C{"ssh.example.com"}, then the port entry for C{"Host *.example.com"}

150

will win out.

151

152

The keys in the returned dict are all normalized to lowercase (look for

153

C{"port"}, not C{"Port"}. No other processing is done to the keys or

154

values.

155

156

@param hostname: the hostname to lookup

157

@type hostname: str

158

"""

159

matches = [

160

x for x in self._config if fnmatch.fnmatch(hostname, x['host'])]

161

# sort in order of shortest match (usually '*') to longest

162

matches.sort(lambda x,y: cmp(len(x['host']), len(y['host'])))

163

ret = {}

164

for m in matches:

165

ret.update(m)

166

del ret['host']

167

return ret

168

169

# END paramiko config fork

170

#############################################################################

171

172

173

def get_ip():

174

"""Uses AWS checkip to obtain this machine's IP address.

175

176

Consults an external website to determine the public IP address of this

177

machine.

178

179

:return: This machine's net-visible IP address as a string.

180

"""

181

return urllib.urlopen('http://checkip.amazonaws.com').read().strip()

182

183

184

class CredentialsError(Exception):

185

"""Raised when AWS credentials could not be loaded."""

186

187

def __init__(self, filename, extra=None):

188

message = (

189

"Please put your aws access key identifier and secret access "

190

"key identifier in %s. (On two lines)." % (filename,))

191

if extra:

192

message += extra

193

Exception.__init__(self, message)

194

195

196

class EC2Credentials:

197

"""Credentials for logging in to EC2."""

198

199

DEFAULT_CREDENTIALS_FILE = '~/.ec2/aws_id'

200

201

def __init__(self, identifier, secret):

202

self.identifier = identifier

203

self.secret = secret

204

205

@classmethod

206

def load_from_file(cls, filename=None):

207

"""Load the EC2 credentials from 'filename'."""

208

if filename is None:

209

filename = os.path.expanduser(cls.DEFAULT_CREDENTIALS_FILE)

210

try:

211

aws_file = open(filename, 'r')

212

except (IOError, OSError), e:

213

raise CredentialsError(filename, str(e))

214

try:

215

identifier = aws_file.readline().strip()

216

secret = aws_file.readline().strip()

217

finally:

218

aws_file.close()

219

return cls(identifier, secret)

220

221

def connect(self, name):

222

"""Connect to EC2 with these credentials.

223

224

:param name: ???

225

:return: An `EC2Account` connected to EC2 with these credentials.

226

"""

227

conn = boto.connect_ec2(self.identifier, self.secret)

228

return EC2Account(name, conn)

229

230

231

class EC2Account:

232

"""An EC2 account.

233

234

You can use this to manage security groups, keys and images for an EC2

235

account.

236

"""

237

238

# Used to find pre-configured Amazon images.

239

_image_match = re.compile(

240

r'launchpad-ec2test(\d+)/image.manifest.xml$').match

241

242

def __init__(self, name, connection):

243

"""Construct an EC2 instance.

244

245

:param name: ???

246

:param connection: An open boto ec2 connection.

247

"""

248

self.name = name

249

self.conn = connection

250

251

def log(self, msg):

252

"""Log a message on stdout, flushing afterwards."""

253

# XXX: JonathanLange 2009-05-31 bug=383076: Copied from EC2TestRunner.

254

# Should change EC2Account to take a logger and use that instead of

255

# writing to stdout.

256

sys.stdout.write(msg)

257

sys.stdout.flush()

258

259

def acquire_security_group(self, demo_networks=None):

260

"""Get a security group with the appropriate configuration.

261

262

"Appropriate" means configured to allow this machine to connect via

263

SSH, HTTP and HTTPS.

264

265

If a group is already configured with this name for this connection,

266

then re-use that. Otherwise, create a new security group and configure

267

it appropriately.

268

269

The name of the security group is the `EC2Account.name` attribute.

270

271

:return: A boto security group.

272

"""

273

if demo_networks is None:

274

demo_networks = []

275

try:

276

group = self.conn.get_all_security_groups(self.name)[0]

277

except EC2ResponseError, e:

278

if e.code != 'InvalidGroup.NotFound':

279

raise

280

else:

281

# If an existing security group was configured, try deleting it

282

# since our external IP might have changed.

283

try:

284

group.delete()

285

except EC2ResponseError, e:

286

if e.code != 'InvalidGroup.InUse':

287

raise

288

# Otherwise, it means that an instance is already using

289

# it, so simply re-use it. It's unlikely that our IP changed!

290

#

291

# XXX: JonathanLange 2009-06-05: If the security group exists

292

# already, verify that the current IP is permitted; if it is

293

# not, make an INFO log and add the current IP.

294

self.log("Security group already in use, so reusing.")

295

return group

296

297

security_group = self.conn.create_security_group(

298

self.name, 'Authorization to access the test runner instance.')

299

# Authorize SSH and HTTP.

300

ip = get_ip()

301

security_group.authorize('tcp', 22, 22, '%s/32' % ip)

302

security_group.authorize('tcp', 80, 80, '%s/32' % ip)

303

security_group.authorize('tcp', 443, 443, '%s/32' % ip)

304

for network in demo_networks:

305

# Add missing netmask info for single ips.

306

if '/' not in network:

307

network += '/32'

308

security_group.authorize('tcp', 80, 80, network)

309

security_group.authorize('tcp', 443, 443, network)

310

return security_group

311

312

def acquire_private_key(self):

313

"""Create & return a new key pair for the test runner."""

314

key_pair = self.conn.create_key_pair(self.name)

315

return paramiko.RSAKey.from_private_key(

316

cStringIO.StringIO(key_pair.material.encode('ascii')))

317

318

def delete_previous_key_pair(self):

319

"""Delete previously used keypair, if it exists."""

320

try:

321

# Only one keypair will match 'self.name' since it's a unique

322

# identifier.

323

key_pairs = self.conn.get_all_key_pairs(self.name)

324

assert len(key_pairs) == 1, (

325

"Should be only one keypair, found %d (%s)"

326

% (len(key_pairs), key_pairs))

327

key_pair = key_pairs[0]

328

key_pair.delete()

329

except EC2ResponseError, e:

330

if e.code != 'InvalidKeyPair.NotFound':

331

if e.code == 'AuthFailure':

332

# Inserted because of previous support issue.

333

self.log(

334

'POSSIBLE CAUSES OF ERROR:\n'

335

' Did you sign up for EC2?\n'

336

' Did you put a credit card number in your AWS '

337

'account?\n'

338

'Please doublecheck before reporting a problem.\n')

339

raise

340

341

def acquire_image(self, machine_id):

342

"""Get the image.

343

344

If 'machine_id' is None, then return the image with location that

345

matches `EC2Account._image_match` and has the highest revision number

346

(where revision number is the 'NN' in 'launchpad-ec2testNN').

347

348

Otherwise, just return the image with the given 'machine_id'.

349

350

:raise ValueError: if there is more than one image with the same

351

location string.

352

353

:raise RuntimeError: if we cannot find a test-runner image.

354

355

:return: A boto image.

356

"""

357

if machine_id is not None:

358

# This may raise an exception. The user specified a machine_id, so

359

# they can deal with it.

360

return self.conn.get_image(machine_id)

361

362

# We are trying to find an image that has a location that matches a

363

# regex (see definition of _image_match, above). Part of that regex is

364

# expected to be an integer with the semantics of a revision number.

365

# The image location with the highest revision number is the one that

366

# should be chosen. Because AWS does not guarantee that two images

367

# cannot share a location string, we need to make sure that the search

368

# result for this image is unique, or throw an error because the

369

# choice of image is ambiguous.

370

search_results = None

371

372

# Find the images with the highest revision numbers and locations that

373

# match the regex.

374

for image in self.conn.get_all_images(owners=VALID_AMI_OWNERS):

375

match = self._image_match(image.location)

376

if match:

377

revision = int(match.group(1))

378

if (search_results is None

379

or search_results['revision'] < revision):

380

# Then we have our first, highest match.

381

search_results = {'revision': revision, 'images': [image]}

382

elif search_results['revision'] == revision:

383

# Another image that matches and is equally high.

384

search_results['images'].append(image)

385

386

# No matching image.

387

if search_results is None:

388

raise RuntimeError(

389

"You don't have access to a test-runner image.\n"

390

"Request access and try again.\n")

391

392

# More than one matching image.

393

if len(search_results['images']) > 1:

394

raise ValueError(

395

('more than one image of revision %(revision)d found: '

396

'%(images)r') % search_results)

397

398

# We could put a minimum image version number check here.

399

image = search_results['images'][0]

400

self.log(

401

'Using machine image version %d\n'

402

% (search_results['revision'],))

403

return image

404

405

def get_instance(self, instance_id):

406

"""Look in all of our reservations for an instance with the given ID.

407

408

Return the instance object if it exists, None otherwise.

409

"""

410

# XXX mars 20090729

411

# This method is needed by the ec2-generate-windmill-image.py script,

412

# so please do not delete it.

413

#

414

# This is a strange object on which to put this method, but I did

415

# not want to break encapsulation around the self.conn attribute.

416

417

for reservation in self.conn.get_all_instances():

418

# We need to look inside each reservation for the instances

419

# themselves.

420

for instance in reservation.instances:

421

if instance.id == instance_id:

422

return instance

423

return None

424

425

426

class UnknownBranchURL(Exception):

427

"""Raised when we try to parse an unrecognized branch url."""

428

429

def __init__(self, branch_url):

430

Exception.__init__(

431

self,

432

"Couldn't parse '%s', not a Launchpad branch." % (branch_url,))

433

434

435

def parse_branch_url(branch_url):

436

"""Given the URL of a branch, return its components in a dict."""

437

_lp_match = re.compile(

438

r'lp:\~([^/]+)/([^/]+)/([^/]+)$').match

439

_bazaar_match = re.compile(

440

r'bzr+ssh://bazaar.launchpad.net/\~([^/]+)/([^/]+)/([^/]+)$').match

441

match = _lp_match(branch_url)

442

if match is None:

443

match = _bazaar_match(branch_url)

444

if match is None:

445

raise UnknownBranchURL(branch_url)

446

owner = match.group(1)

447

product = match.group(2)

448

branch = match.group(3)

449

unique_name = '~%s/%s/%s' % (owner, product, branch)

450

url = 'bzr+ssh://bazaar.launchpad.net/%s' % (unique_name,)

451

return dict(

452

owner=owner, product=product, branch=branch, unique_name=unique_name,

453

url=url)

454

455

456

def validate_file(filename):

457

"""Raise an error if 'filename' is not a file we can write to."""

458

if filename is None:

459

return

460

461

check_file = filename

462

if os.path.exists(check_file):

463

if not os.path.isfile(check_file):

464

raise ValueError(

465

'file argument %s exists and is not a file' % (filename,))

466

else:

467

check_file = os.path.dirname(check_file)

468

if (not os.path.exists(check_file) or

469

not os.path.isdir(check_file)):

470

raise ValueError(

471

'file %s cannot be created.' % (filename,))

472

if not os.access(check_file, os.W_OK):

473

raise ValueError(

474

'you do not have permission to write %s' % (filename,))

475

476

477

def normalize_branch_input(data):

478

"""Given 'data' return a ('dest', 'src') pair.

479

480

:param data: One of::

481

- a double of (sourcecode_location, branch_url).

482

If 'sourcecode_location' is Launchpad, then 'branch_url' can

483

also be the name of a branch of launchpad owned by

484

launchpad-pqm.

485

- a singleton of (branch_url,)

486

- a singleton of (sourcecode_location,) where

487

sourcecode_location corresponds to a Launchpad upstream

488

project as well as a rocketfuel sourcecode location.

489

- a string which could populate any of the above singletons.

490

491

:return: ('dest', 'src') where 'dest' is the destination

492

sourcecode location in the rocketfuel tree and 'src' is the

493

URL of the branch to put there. The URL can be either a bzr+ssh

494

URL or the name of a branch of launchpad owned by launchpad-pqm.

495

"""

496

# XXX: JonathanLange 2009-06-05: Should convert lp: URL branches to

497

# bzr+ssh:// branches.

498

if isinstance(data, basestring):

499

data = (data,)

500

if len(data) == 2:

501

# Already in dest, src format.

502

return data

503

if len(data) != 1:

504

raise ValueError(

505

'invalid argument for ``branches`` argument: %r' %

506

(data,))

507

branch_location = data[0]

508

try:

509

parsed_url = parse_branch_url(branch_location)

510

except UnknownBranchURL:

511

return branch_location, 'lp:%s' % (branch_location,)

512

return parsed_url['product'], parsed_url['url']

513

514

515

def parse_specified_branches(branches):

516

"""Given 'branches' from the command line, return a sanitized dict.

517

518

The dict maps sourcecode locations to branch URLs, according to the

519

rules in `normalize_branch_input`.

520

"""

521

return dict(map(normalize_branch_input, branches))

522

523

524

class EC2Instance:

525

"""A single EC2 instance."""

526

527

# XXX: JonathanLange 2009-05-31: Make it so that we pass one of these to

528

# EC2 test runner, rather than the test runner knowing how to make one.

529

# Right now, the test runner makes one of these directly. Instead, we want

530

# to make an EC2Account and ask it for one of these instances and then

531

# pass it to the test runner on construction.

532

533

# XXX: JonathanLange 2009-05-31: Separate out demo server maybe?

534

535

# XXX: JonathanLange 2009-05-31: Possibly separate out "get an instance"

536

# and "set up instance for Launchpad testing" logic.

537

538

def __init__(self, name, image, instance_type, demo_networks, controller,

539

vals):

540

self._name = name

541

self._image = image

542

self._controller = controller

543

self._instance_type = instance_type

544

self._demo_networks = demo_networks

545

self._boto_instance = None

546

self._vals = vals

547

548

def error_and_quit(self, msg):

549

"""Print error message and exit."""

550

sys.stderr.write(msg)

551

sys.exit(1)

552

553

def log(self, msg):

554

"""Log a message on stdout, flushing afterwards."""

555

# XXX: JonathanLange 2009-05-31 bug=383076: Should delete this and use

556

# Python logging module instead.

557

sys.stdout.write(msg)

558

sys.stdout.flush()

559

560

def start(self):

561

"""Start the instance."""

562

if self._boto_instance is not None:

563

self.log('Instance %s already started' % self._boto_instance.id)

564

return

565

start = time.time()

566

self.private_key = self._controller.acquire_private_key()

567

self._controller.acquire_security_group(

568

demo_networks=self._demo_networks)

569

reservation = self._image.run(

570

key_name=self._name, security_groups=[self._name],

571

instance_type=self._instance_type)

572

self._boto_instance = reservation.instances[0]

573

self.log('Instance %s starting..' % self._boto_instance.id)

574

while self._boto_instance.state == 'pending':

575

self.log('.')

576

time.sleep(5)

577

self._boto_instance.update()

578

if self._boto_instance.state == 'running':

579

self.log(' started on %s\n' % self.hostname)

580

elapsed = time.time() - start

581

self.log('Started in %d minutes %d seconds\n' %

582

(elapsed // 60, elapsed % 60))

583

self._output = self._boto_instance.get_console_output()

584

self.log(self._output.output)

585

else:

586

self.error_and_quit(

587

'failed to start: %s\n' % self._boto_instance.state)

588

589

def shutdown(self):

590

"""Shut down the instance."""

591

if self._boto_instance is None:

592

self.log('no instance created\n')

593

return

594

self._boto_instance.update()

595

if self._boto_instance.state not in ('shutting-down', 'terminated'):

596

# terminate instance

597

self._boto_instance.stop()

598

self._boto_instance.update()

599

self.log('instance %s\n' % (self._boto_instance.state,))

600

601

@property

602

def hostname(self):

603

if self._boto_instance is None:

604

return None

605

return self._boto_instance.public_dns_name

606

607

def connect_as_root(self):

608

"""Connect to the instance as root.

609

610

All subsequent 'perform' and 'subprocess' operations will be done with

611

root privileges.

612

"""

613

# XXX: JonathanLange 2009-06-02: This state-changing method could

614

# perhaps be written as a function such as run_as_root, or as a method

615

# that returns a root connection.

616

for count in range(10):

617

self.ssh = paramiko.SSHClient()

618

self.ssh.set_missing_host_key_policy(AcceptAllPolicy())

619

self.username = 'root'

620

try:

621

self.ssh.connect(

622

self.hostname, username='root',

623

pkey=self.private_key,

624

allow_agent=False, look_for_keys=False)

625

except (socket.error, paramiko.AuthenticationException), e:

626

self.log('connect_as_root: %r' % (e,))

627

if count < 9:

628

time.sleep(5)

629

self.log('retrying...')

630

else:

631

raise

632

else:

633

break

634

635

def connect_as_user(self):

636

"""Connect as user.

637

638

All subsequent 'perform' and 'subprocess' operations will be done with

639

user-level privileges.

640

"""

641

# XXX: JonathanLange 2009-06-02: This state-changing method could

642

# perhaps be written as a function such as run_as_user, or as a method

643

# that returns a user connection.

644

#

645

# This does not have the retry logic of connect_as_root because the

646

# circumstances that make the retries necessary appear to only happen

647

# on start-up, and connect_as_root is called first.

648

self.ssh = paramiko.SSHClient()

649

self.ssh.set_missing_host_key_policy(AcceptAllPolicy())

650

self.username = self._vals['USER']

651

self.ssh.connect(self.hostname)

652

653

def perform(self, cmd, ignore_failure=False, out=None):

654

"""Perform 'cmd' on server.

655

656

:param ignore_failure: If False, raise an error on non-zero exit

657

statuses.

658

:param out: A stream to write the output of the remote command to.

659

"""

660

cmd = cmd % self._vals

661

self.log('%s@%s$ %s\n' % (self.username, self._boto_instance.id, cmd))

662

session = self.ssh.get_transport().open_session()

663

session.exec_command(cmd)

664

session.shutdown_write()

665

while 1:

666

select.select([session], [], [], 0.5)

667

if session.recv_ready():

668

data = session.recv(4096)

669

if data:

670

sys.stdout.write(data)

671

sys.stdout.flush()

672

if out is not None:

673

out.write(data)

674

if session.recv_stderr_ready():

675

data = session.recv_stderr(4096)

676

if data:

677

sys.stderr.write(data)

678

sys.stderr.flush()

679

if session.exit_status_ready():

680

break

681

session.close()

682

# XXX: JonathanLange 2009-05-31: If the command is killed by a signal

683

# on the remote server, the SSH protocol does not send an exit_status,

684

# it instead sends a different message with the number of the signal

685

# that killed the process. AIUI, this code will fail confusingly if

686

# that happens.

687

res = session.recv_exit_status()

688

if res and not ignore_failure:

689

raise RuntimeError('Command failed: %s' % (cmd,))

690

return res

691

692

def run_with_ssh_agent(self, cmd, ignore_failure=False):

693

"""Run 'cmd' in a subprocess.

694

695

Use this to run commands that require local SSH credentials. For

696

example, getting private branches from Launchpad.

697

"""

698

cmd = cmd % self._vals

699

self.log('%s@%s$ %s\n' % (self.username, self._boto_instance.id, cmd))

700

call = ['ssh', '-A', self.hostname,

701

'-o', 'CheckHostIP no',

702

'-o', 'StrictHostKeyChecking no',

703

'-o', 'UserKnownHostsFile ~/.ec2/known_hosts',

704

cmd]

705

res = subprocess.call(call)

706

if res and not ignore_failure:

707

raise RuntimeError('Command failed: %s' % (cmd,))

708

return res

709

710

711

class EC2TestRunner:

712

713

name = 'ec2-test-runner'

714

715

message = instance = image = None

716

_running = False

717

718

def __init__(self, branch, email=False, file=None, test_options='-vv',

719

headless=False, branches=(),

720

machine_id=None, instance_type=DEFAULT_INSTANCE_TYPE,

721

pqm_message=None, pqm_public_location=None,

722

pqm_submit_location=None, demo_networks=None,

723

open_browser=False, pqm_email=None,

724

include_download_cache_changes=None):

725

"""Create a new EC2TestRunner.

726

727

This sets the following attributes:

728

- original_branch

729

- test_options

730

- headless

731

- include_download_cache_changes

732

- download_cache_additions

733

- branches (parses, validates)

734

- message (after validating PQM submisson)

735

- email (after validating email capabilities)

736

- instance_type (validates)

737

- image (after connecting to ec2)

738

- file (after checking we can write to it)

739

- ssh_config_file_name (after checking it exists)

740

- vals, a dict containing

741

- the environment

742

- trunk_branch (either from global or derived from branches)

743

- branch

744

- smtp_server

745

- smtp_username

746

- smtp_password

747

- email (distinct from the email attribute)

748

- key_type

749

- key

750

- launchpad_login

751

"""

752

self.original_branch = branch # just for easy access in debugging

753

self.test_options = test_options

754

self.headless = headless

755

self.include_download_cache_changes = include_download_cache_changes

756

if demo_networks is None:

757

demo_networks = ()

758

else:

759

demo_networks = demo_networks

760

self.open_browser = open_browser

761

if headless and file:

762

raise ValueError(

763

'currently do not support files with headless mode.')

764

if headless and not (email or pqm_message):

765

raise ValueError('You have specified no way to get the results '

766

'of your headless test run.')

767

768

if test_options != '-vv' and pqm_message is not None:

769

raise ValueError(

770

"Submitting to PQM with non-default test options isn't "

771

"supported")

772

773

trunk_specified = False

774

trunk_branch = TRUNK_BRANCH

775

776

# normalize and validate branches

777

branches = parse_specified_branches(branches)

778

try:

779

launchpad_url = branches.pop('launchpad')

780

except KeyError:

781

# No Launchpad branch specified.

782

pass

783

else:

784

try:

785

parsed_url = parse_branch_url(launchpad_url)

786

except UnknownBranchURL:

787

user = 'launchpad-pqm'

788

src = ('bzr+ssh://bazaar.launchpad.net/'

789

'~launchpad-pqm/launchpad/%s' % (launchpad_url,))

790

else:

791

user = parsed_url['owner']

792

src = parsed_url['url']

793

if user == 'launchpad-pqm':

794

trunk_specified = True

795

trunk_branch = src

796

797

self.branches = branches.items()

798

799

# XXX: JonathanLange 2009-05-31: The trunk_specified stuff above and

800

# the pqm location stuff below are actually doing the equivalent of

801

# preparing a merge directive. Perhaps we can leverage that to make

802

# this code simpler.

803

self.download_cache_additions = None

804

if branch is None:

805

config = GlobalConfig()

806

if pqm_message is not None:

807

raise ValueError('Cannot submit trunk to pqm.')

808

else:

809

(tree,

810

bzrbranch,

811

relpath) = BzrDir.open_containing_tree_or_branch(branch)

812

# if tree is None, remote...I'm assuming.

813

if tree is None:

814

config = GlobalConfig()

815

else:

816

config = bzrbranch.get_config()

817

818

if pqm_message is not None or tree is not None:

819

# if we are going to maybe send a pqm_message, we're going to

820

# go down this path. Also, even if we are not but this is a

821

# local branch, we're going to use the PQM machinery to make

822

# sure that the local branch has been made public, and has all

823

# working changes there.

824

if tree is None:

825

# remote. We will make some assumptions.

826

if pqm_public_location is None:

827

pqm_public_location = branch

828

if pqm_submit_location is None:

829

pqm_submit_location = trunk_branch

830

elif pqm_submit_location is None and trunk_specified:

831

pqm_submit_location = trunk_branch

832

# modified from pqm_submit.py

833

submission = PQMSubmission(

834

source_branch=bzrbranch,

835

public_location=pqm_public_location,

836

message=pqm_message or '',

837

submit_location=pqm_submit_location,

838

tree=tree)

839

if tree is not None:

840

# this is the part we want to do whether or not we're

841

# submitting.

842

submission.check_tree() # any working changes

843

submission.check_public_branch() # everything public

844

branch = submission.public_location

845

if (include_download_cache_changes is None or

846

include_download_cache_changes):

847

# We need to get the download cache settings

848

cache_tree, cache_bzrbranch, cache_relpath = (

849

BzrDir.open_containing_tree_or_branch(

850

os.path.join(

851

self.original_branch, 'download-cache')))

852

cache_tree.lock_read()

853

try:

854

cache_basis_tree = cache_tree.basis_tree()

855

cache_basis_tree.lock_read()

856

try:

857

delta = cache_tree.changes_from(

858

cache_basis_tree, want_unversioned=True)

859

unversioned = [

860

un for un in delta.unversioned

861

if not cache_tree.is_ignored(un[0])]

862

added = delta.added

863

self.download_cache_additions = (

864

unversioned + added)

865

finally:

866

cache_basis_tree.unlock()

867

finally:

868

cache_tree.unlock()

869

if pqm_message is not None:

870

if self.download_cache_additions:

871

raise UncommittedChanges(cache_tree)

872

# get the submission message

873

mail_from = config.get_user_option('pqm_user_email')

874

if not mail_from:

875

mail_from = config.username()

876

# Make sure this isn't unicode

877

mail_from = mail_from.encode('utf8')

878

if pqm_email is None:

879

if tree is None:

880

pqm_email = (

881

"Launchpad PQM <launchpad@pqm.canonical.com>")

882

else:

883

pqm_email = config.get_user_option('pqm_email')

884

if not pqm_email:

885

raise NoPQMSubmissionAddress(bzrbranch)

886

mail_to = pqm_email.encode('utf8') # same here

887

self.message = submission.to_email(mail_from, mail_to)

888

elif (self.download_cache_additions and

889

self.include_download_cache_changes is None):

890

raise UncommittedChanges(

891

cache_tree,

892

'You must select whether to include download cache '

893

'changes (see --include-download-cache-changes and '

894

'--ignore-download-cache-changes, -c and -g '

895

'respectively), or '

896

'commit or remove the files in the download-cache.')

897

if email is not False:

898

if email is True:

899

email = [config.username()]

900

if not email[0]:

901

raise ValueError('cannot find your email address.')

902

elif isinstance(email, basestring):

903

email = [email]

904

else:

905

tmp = []

906

for item in email:

907

if not isinstance(item, basestring):

908

raise ValueError(

909

'email must be True, False, a string, or a list of '

910

'strings')

911

tmp.append(item)

912

email = tmp

913

else:

914

email = None

915

self.email = email

916

917

# We do a lot of looking before leaping here because we want to avoid

918

# wasting time and money on errors we could have caught early.

919

920

# Validate instance_type and get default kernal and ramdisk.

921

if instance_type not in AVAILABLE_INSTANCE_TYPES:

922

raise ValueError('unknown instance_type %s' % (instance_type,))

923

924

# Validate and set file.

925

validate_file(file)

926

self.file = file

927

928

# Make a dict for string substitution based on the environ.

929

#

930

# XXX: JonathanLange 2009-06-02: Although this defintely makes the

931

# scripts & commands easier to write, it makes it harder to figure out

932

# how the different bits of the system interoperate (passing 'vals' to

933

# a method means it uses...?). Consider changing things around so that

934

# vals is not needed.

935

self.vals = dict(os.environ)

936

self.vals['trunk_branch'] = trunk_branch

937

self.vals['branch'] = branch

938

home = self.vals['HOME']

939

940

# Email configuration.

941

if email is not None or pqm_message is not None:

942

server = self.vals['smtp_server'] = config.get_user_option(

943

'smtp_server')

944

if server is None or server == 'localhost':

945

raise ValueError(

946

'To send email, a remotely accessible smtp_server (and '

947

'smtp_username and smtp_password, if necessary) must be '

948

'configured in bzr. See the SMTP server information '

949

'here: https://wiki.canonical.com/EmailSetup .')

950

self.vals['smtp_username'] = config.get_user_option(

951

'smtp_username')

952

self.vals['smtp_password'] = config.get_user_option(

953

'smtp_password')

954

from_email = config.username()

955

if not from_email:

956

raise ValueError(

957

'To send email, your bzr email address must be set '

958

'(use ``bzr whoami``).')

959

else:

960

self.vals['email'] = (

961

from_email.encode('utf8').encode('string-escape'))

962

963

# Get a public key from the agent.

964

agent = paramiko.Agent()

965

keys = agent.get_keys()

966

if len(keys) == 0:

967

self.error_and_quit(

968

'You must have an ssh agent running with keys installed that '

969

'will allow the script to rsync to devpad and get your '

970

'branch.\n')

971

key = agent.get_keys()[0]

972

self.vals['key_type'] = key.get_name()

973

self.vals['key'] = key.get_base64()

974

975

# Verify the .ssh config file

976

self.ssh_config_file_name = os.path.join(home, '.ssh', 'config')

977

if not os.path.exists(self.ssh_config_file_name):

978

self.error_and_quit(

979

'This script expects to find the .ssh config in %s. Please '

980

'make sure it exists and contains the necessary '

981

'configuration to access devpad.' % (

982

self.ssh_config_file_name,))

983

984

# Get the bzr login.

985

login = get_lp_login()

986

if not login:

987

self.error_and_quit(

988

'you must have set your launchpad login in bzr.')

989

self.vals['launchpad-login'] = login

990

991

# Get the AWS identifier and secret identifier.

992

try:

993

credentials = EC2Credentials.load_from_file()

994

except CredentialsError, e:

995

self.error_and_quit(str(e))

996

997

# Make the EC2 connection.

998

controller = credentials.connect(self.name)

999

1000

# We do this here because it (1) cleans things up and (2) verifies

1001

# that the account is correctly set up. Both of these are appropriate

1002

# for initialization.

1003

#

1004

# We always recreate the keypairs because there is no way to

1005

# programmatically retrieve the private key component, unless we

1006

# generate it.

1007

controller.delete_previous_key_pair()

1008

1009

# get the image

1010

image = controller.acquire_image(machine_id)

1011

self._instance = EC2Instance(

1012

self.name, image, instance_type, demo_networks,

1013

controller, self.vals)

1014

# now, as best as we can tell, we should be good to go.

1015

1016

def error_and_quit(self, msg):

1017

"""Print error message and exit."""

1018

sys.stderr.write(msg)

1019

sys.exit(1)

1020

1021

def log(self, msg):

1022

"""Log a message on stdout, flushing afterwards."""

1023

# XXX: JonathanLange 2009-05-31 bug=383076: This should use Python

1024

# logging, rather than printing to stdout.

1025

sys.stdout.write(msg)

1026

sys.stdout.flush()

1027

1028

def start(self):

1029

"""Start the EC2 instance."""

1030

self._instance.start()

1031

1032

def shutdown(self):

1033

if self.headless and self._running:

1034

self.log('letting instance run, to shut down headlessly '

1035

'at completion of tests.\n')

1036

return

1037

return self._instance.shutdown()

1038

1039

def configure_system(self):

1040

# AS ROOT

1041

self._instance.connect_as_root()

1042

if self.vals['USER'] == 'gary':

1043

# This helps gary debug problems others are having by removing

1044

# much of the initial setup used to work on the original image.

1045

self._instance.perform('deluser --remove-home gary',

1046

ignore_failure=True)

1047

p = self._instance.perform

1048

# Let root perform sudo without a password.

1049

p('echo "root\tALL=NOPASSWD: ALL" >> /etc/sudoers')

1050

# Add the user.

1051

p('adduser --gecos "" --disabled-password %(USER)s')

1052

# Give user sudo without password.

1053

p('echo "%(USER)s\tALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers')

1054

# Make /var/launchpad owned by user.

1055

p('chown -R %(USER)s:%(USER)s /var/launchpad')

1056

# Clean out left-overs from the instance image.

1057

p('rm -fr /var/tmp/*')

1058

# Update the system.

1059

p('aptitude update')

1060

p('aptitude -y full-upgrade')

1061

# Set up ssh for user

1062

# Make user's .ssh directory

1063

p('sudo -u %(USER)s mkdir /home/%(USER)s/.ssh')

1064

sftp = self._instance.ssh.open_sftp()

1065

remote_ssh_dir = '/home/%(USER)s/.ssh' % self.vals

1066

# Create config file

1067

self.log('Creating %s/config\n' % (remote_ssh_dir,))

1068

ssh_config_source = open(self.ssh_config_file_name)

1069

config = SSHConfig()

1070

config.parse(ssh_config_source)

1071

ssh_config_source.close()

1072

ssh_config_dest = sftp.open("%s/config" % remote_ssh_dir, 'w')

1073

ssh_config_dest.write('CheckHostIP no\n')

1074

ssh_config_dest.write('StrictHostKeyChecking no\n')

1075

for hostname in ('devpad.canonical.com', 'chinstrap.canonical.com'):

1076

ssh_config_dest.write('Host %s\n' % (hostname,))

1077

data = config.lookup(hostname)

1078

for key in ('hostname', 'gssapiauthentication', 'proxycommand',

1079

'user', 'forwardagent'):

1080

value = data.get(key)

1081

if value is not None:

1082

ssh_config_dest.write(' %s %s\n' % (key, value))

1083

ssh_config_dest.write('Host bazaar.launchpad.net\n')

1084

ssh_config_dest.write(' user %(launchpad-login)s\n' % self.vals)

1085

ssh_config_dest.close()

1086

# create authorized_keys

1087

self.log('Setting up %s/authorized_keys\n' % remote_ssh_dir)

1088

authorized_keys_file = sftp.open(

1089

"%s/authorized_keys" % remote_ssh_dir, 'w')

1090

authorized_keys_file.write("%(key_type)s %(key)s\n" % self.vals)

1091

authorized_keys_file.close()

1092

sftp.close()

1093

# Chown and chmod the .ssh directory and contents that we just

1094

# created.

1095

p('chown -R %(USER)s:%(USER)s /home/%(USER)s/')

1096

p('chmod 644 /home/%(USER)s/.ssh/*')

1097

self.log(

1098

'You can now use ssh -A %s to log in the instance.\n' %

1099

self._instance.hostname)

1100

# give the user permission to do whatever in /var/www

1101

p('chown -R %(USER)s:%(USER)s /var/www')

1102

self._instance.ssh.close()

1103

1104

# AS USER

1105

self._instance.connect_as_user()

1106

sftp = self._instance.ssh.open_sftp()

1107

# Set up bazaar.conf with smtp information if necessary

1108

if self.email or self.message:

1109

p('sudo -u %(USER)s mkdir /home/%(USER)s/.bazaar')

1110

bazaar_conf_file = sftp.open(

1111

"/home/%(USER)s/.bazaar/bazaar.conf" % self.vals, 'w')

1112

bazaar_conf_file.write(

1113

'smtp_server = %(smtp_server)s\n' % self.vals)

1114

if self.vals['smtp_username']:

1115

bazaar_conf_file.write(

1116

'smtp_username = %(smtp_username)s\n' % self.vals)

1117

if self.vals['smtp_password']:

1118

bazaar_conf_file.write(

1119

'smtp_password = %(smtp_password)s\n' % self.vals)

1120

bazaar_conf_file.close()

1121

# Copy remote ec2-remote over

1122

self.log('Copying ec2test-remote.py to remote machine.\n')

1123

sftp.put(

1124

os.path.join(os.path.dirname(os.path.realpath(__file__)),

1125

'ec2test-remote.py'),

1126

'/var/launchpad/ec2test-remote.py')

1127

sftp.close()

1128

# Set up launchpad login and email

1129

p('bzr launchpad-login %(launchpad-login)s')

1130

p("bzr whoami '%(email)s'")

1131

self._instance.ssh.close()

1132

1133

def prepare_tests(self):

1134

self._instance.connect_as_user()

1135

# Clean up the test branch left in the instance image.

1136

self._instance.perform('rm -rf /var/launchpad/test')

1137

# get newest sources

1138

self._instance.run_with_ssh_agent(

1139

"rsync -avp --partial --delete "

1140

"--filter='P *.o' --filter='P *.pyc' --filter='P *.so' "

1141

"devpad.canonical.com:/code/rocketfuel-built/launchpad/sourcecode/* "

1142

"/var/launchpad/sourcecode/")

1143

# Get trunk.

1144

self._instance.run_with_ssh_agent(

1145

'bzr branch %(trunk_branch)s /var/launchpad/test')

1146

# Merge the branch in.

1147

if self.vals['branch'] is not None:

1148

self._instance.run_with_ssh_agent(

1149

'cd /var/launchpad/test; bzr merge %(branch)s')

1150

else:

1151

self.log('(Testing trunk, so no branch merge.)')

1152

# Get any new sourcecode branches as requested

1153

for dest, src in self.branches:

1154

fulldest = os.path.join('/var/launchpad/test/sourcecode', dest)

1155

if dest in ('canonical-identity-provider', 'shipit'):

1156

# These two branches share some of the history with Launchpad.

1157

# So we create a stacked branch on Launchpad so that the shared

1158

# history isn't duplicated.

1159

self._instance.run_with_ssh_agent(

1160

'bzr branch --no-tree --stacked %s %s' %

1161

(TRUNK_BRANCH, fulldest))

1162

# The --overwrite is needed because they are actually two

1163

# different branches (canonical-identity-provider was not

1164

# branched off launchpad, but some revisions are shared.)

1165

self._instance.run_with_ssh_agent(

1166

'bzr pull --overwrite %s -d %s' % (src, fulldest))

1167

# The third line is necessary because of the --no-tree option

1168

# used initially. --no-tree doesn't create a working tree.

1169

# It only works with the .bzr directory (branch metadata and

1170

# revisions history). The third line creates a working tree

1171

# based on the actual branch.

1172

self._instance.run_with_ssh_agent(

1173

'bzr checkout "%s" "%s"' % (fulldest, fulldest))

1174

else:

1175

# The "--standalone" option is needed because some branches

1176

# are/were using a different repository format than Launchpad

1177

# (bzr-svn branch for example).

1178

self._instance.run_with_ssh_agent(

1179

'bzr branch --standalone %s %s' % (src, fulldest))

1180

# prepare fresh copy of sourcecode and buildout sources for building

1181

p = self._instance.perform

1182

p('rm -rf /var/launchpad/tmp')

1183

p('mkdir /var/launchpad/tmp')

1184

p('cp -R /var/launchpad/sourcecode /var/launchpad/tmp/sourcecode')

1185

p('mkdir /var/launchpad/tmp/eggs')

1186

self._instance.run_with_ssh_agent(

1187

'bzr co lp:lp-source-dependencies '

1188

'/var/launchpad/tmp/download-cache')

1189

if (self.include_download_cache_changes and

1190

self.download_cache_additions):

1191

sftp = self._instance.ssh.open_sftp()

1192

root = os.path.realpath(

1193

os.path.join(self.original_branch, 'download-cache'))

1194

for info in self.download_cache_additions:

1195

src = os.path.join(root, info[0])

1196

self.log('Copying %s to remote machine.\n' % (src,))

1197

sftp.put(

1198

src,

1199

os.path.join('/var/launchpad/tmp/download-cache', info[0]))

1200

sftp.close()

1201

p('/var/launchpad/test/utilities/link-external-sourcecode '

1202

'-p/var/launchpad/tmp -t/var/launchpad/test'),

1203

# set up database

1204

p('/var/launchpad/test/utilities/launchpad-database-setup %(USER)s')

1205

p('cd /var/launchpad/test && make build')

1206

p('cd /var/launchpad/test && make schema')

1207

# close ssh connection

1208

self._instance.ssh.close()

1209

1210

def start_demo_webserver(self):

1211

"""Turn ec2 instance into a demo server."""

1212

self._instance.connect_as_user()

1213

p = self._instance.perform

1214

p('mkdir -p /var/tmp/bazaar.launchpad.dev/static')

1215

p('mkdir -p /var/tmp/bazaar.launchpad.dev/mirrors')

1216

p('sudo a2enmod proxy > /dev/null')

1217

p('sudo a2enmod proxy_http > /dev/null')

1218

p('sudo a2enmod rewrite > /dev/null')

1219

p('sudo a2enmod ssl > /dev/null')

1220

p('sudo a2enmod deflate > /dev/null')

1221

p('sudo a2enmod headers > /dev/null')

1222

# Install apache config file.

1223

p('cd /var/launchpad/test/; sudo make install')

1224

# Use raw string to eliminate the need to escape the backslash.

1225

# Put eth0's ip address in the /tmp/ip file.

1226

p(r"ifconfig eth0 | grep 'inet addr' "

1227

r"| sed -re 's/.*addr:([0-9.]*) .*/\1/' > /tmp/ip")

1228

# Replace 127.0.0.88 in Launchpad's apache config file with the

1229

# ip address just stored in the /tmp/ip file. Perl allows for

1230

# inplace editing unlike sed.

1231

p('sudo perl -pi -e "s/127.0.0.88/$(cat /tmp/ip)/g" '

1232

'/etc/apache2/sites-available/local-launchpad')

1233

# Restart apache.

1234

p('sudo /etc/init.d/apache2 restart')

1235

# Build mailman and minified javascript, etc.

1236

p('cd /var/launchpad/test/; make')

1237

# Start launchpad in the background.

1238

p('cd /var/launchpad/test/; make start')

1239

# close ssh connection

1240

self._instance.ssh.close()

1241

1242

def run_tests(self):

1243

self._instance.connect_as_user()

1244

1245

# Make sure we activate the failsafe --shutdown feature. This will

1246

# make the server shut itself down after the test run completes, or

1247

# if the test harness suffers a critical failure.

1248

cmd = ['python /var/launchpad/ec2test-remote.py --shutdown']

1249

1250

# Do we want to email the results to the user?

1251

if self.email:

1252

for email in self.email:

1253

cmd.append("--email='%s'" % (

1254

email.encode('utf8').encode('string-escape'),))

1255

1256

# Do we want to submit the branch to PQM if the tests pass?

1257

if self.message is not None:

1258

cmd.append(

1259

"--submit-pqm-message='%s'" % (

1260

pickle.dumps(

1261

self.message).encode(

1262

'base64').encode('string-escape'),))

1263

1264

# Do we want to disconnect the terminal once the test run starts?

1265

if self.headless:

1266

cmd.append('--daemon')

1267

1268

# Which branch do we want to test?

1269

if self.vals['branch'] is not None:

1270

branch = self.vals['branch']

1271

remote_branch = Branch.open(branch)

1272

branch_revno = remote_branch.revno()

1273

else:

1274

branch = self.vals['trunk_branch']

1275

branch_revno = None

1276

cmd.append('--public-branch=%s' % branch)

1277

if branch_revno is not None:

1278

cmd.append('--public-branch-revno=%d' % branch_revno)

1279

1280

# Add any additional options for ec2test-remote.py

1281

cmd.extend(self.get_remote_test_options())

1282

self.log(

1283

'Running tests... (output is available on '

1284

'http://%s/)\n' % self._instance.hostname)

1285

1286

# Try opening a browser pointed at the current test results.

1287

if self.open_browser:

1288

try:

1289

import webbrowser

1290

except ImportError:

1291

self.log("Could not open web browser due to ImportError.")

1292

else:

1293

status = webbrowser.open(self._instance.hostname)

1294

if not status:

1295

self.log("Could not open web browser.")

1296

1297

# Run the remote script! Our execution will block here until the

1298

# remote side disconnects from the terminal.

1299

self._instance.perform(' '.join(cmd))

1300

self._running = True

1301

1302

if not self.headless:

1303

sftp = self._instance.ssh.open_sftp()

1304

# We ran to completion locally, so we'll be in charge of shutting

1305

# down the instance, in case the user has requested a postmortem.

1306

#

1307

# We only have 60 seconds to do this before the remote test

1308

# script shuts the server down automatically.

1309

self._instance.perform(

1310

'kill `cat /var/launchpad/ec2test-remote.pid`')

1311

1312

# deliver results as requested

1313

if self.file:

1314

self.log(

1315

'Writing abridged test results to %s.\n' % self.file)

1316

sftp.get('/var/www/summary.log', self.file)

1317

sftp.close()

1318

# close ssh connection

1319

self._instance.ssh.close()

1320

1321

def get_remote_test_options(self):

1322

"""Return the test command that will be passed to ec2test-remote.py.

1323

1324

Returns a tuple of command-line options and switches.

1325

"""

1326

if '--jscheck' in self.test_options:

1327

# We want to run the JavaScript test suite.

1328

return ('--jscheck',)

1329

else:

1330

# Run the normal testsuite with our Zope testrunner options.

1331

# ec2test-remote.py wants the extra options to be after a double-

1332

# dash.

1333

return ('--', self.test_options)

1334

1335

1336

1337

class AcceptAllPolicy:

1338

"""We accept all unknown host key."""

1339

1340

# Normally the console output is supposed to contain the Host key

1341

# but it doesn't seem to be the case here, so we trust that the host

1342

# we are connecting to is the correct one.

1343

def missing_host_key(self, client, hostname, key):

1344

pass

1345

1346

1347

# XXX: JonathanLange 2009-05-31: Strongly considering turning this into a

1348

# Bazaar plugin -- probably would make the option parsing and validation

1349

# easier.

1350

1351

if __name__ == '__main__':

1352

parser = optparse.OptionParser(

1353

usage="%prog [options] [branch]",

1354

description=(

1355

"Check out a Launchpad branch and run all tests on an Amazon "

1356

"EC2 instance."))

1357

parser.add_option(

1358

'-f', '--file', dest='file', default=None,

1359

help=('Store abridged test results in FILE.'))

1360

parser.add_option(

1361

'-n', '--no-email', dest='no_email', default=False,

1362

action='store_true',

1363

help=('Do not try to email results.'))

1364

parser.add_option(

1365

'-e', '--email', action='append', dest='email', default=None,

1366

help=('Email address to which results should be mailed. Defaults to '

1367

'the email address from `bzr whoami`. May be supplied multiple '

1368

'times. The first supplied email address will be used as the '

1369

'From: address.'))

1370

parser.add_option(

1371

'-o', '--test-options', dest='test_options', default='-vv',

1372

help=('Test options to pass to the remote test runner. Defaults to '

1373

"``-o '-vv'``. For instance, to run specific tests, you might "

1374

"use ``-o '-vvt my_test_pattern'``."))

1375

parser.add_option(

1376

'-b', '--branch', action='append', dest='branches',

1377

help=('Branches to include in this run in sourcecode. '

1378

'If the argument is only the project name, the trunk will be '

1379

'used (e.g., ``-b launchpadlib``). If you want to use a '

1380

'specific branch, if it is on launchpad, you can usually '

1381

'simply specify it instead (e.g., '

1382

'``-b lp:~username/launchpadlib/branchname``). If this does '

1383

'not appear to work, or if the desired branch is not on '

1384

'launchpad, specify the project name and then the branch '

1385

'after an equals sign (e.g., '

1386

'``-b launchpadlib=lp:~username/launchpadlib/branchname``). '

1387

'Branches for multiple projects may be specified with '

1388

'multiple instances of this option. '

1389

'You may also use this option to specify the branch of launchpad '

1390

'into which your branch may be merged. This defaults to %s. '

1391

'Because typically the important branches of launchpad are owned '

1392

'by the launchpad-pqm user, you can shorten this to only the '

1393

'branch name, if desired, and the launchpad-pqm user will be '

1394

'assumed. For instance, if you specify '

1395

'``-b launchpad=db-devel`` then this is equivalent to '

1396

'``-b lp:~launchpad-pqm/launchpad/db-devel``, or the even longer'

1397

'``-b launchpad=lp:~launchpad-pqm/launchpad/db-devel``.'

1398

% (TRUNK_BRANCH,)))

1399

parser.add_option(

1400

'-t', '--trunk', dest='trunk', default=False,

1401

action='store_true',

1402

help=('Run the trunk as the branch'))

1403

parser.add_option(

1404

'-s', '--submit-pqm-message', dest='pqm_message', default=None,

1405

help=('A pqm message to submit if the test run is successful. If '

1406

'provided, you will be asked for your GPG passphrase before '

1407

'the test run begins.'))

1408

parser.add_option(

1409

'--pqm-public-location', dest='pqm_public_location', default=None,

1410

help=('The public location for the pqm submit, if a pqm message is '

1411

'provided (see --submit-pqm-message). If this is not provided, '

1412

'for local branches, bzr configuration is consulted; for '

1413

'remote branches, it is assumed that the remote branch *is* '

1414

'a public branch.'))

1415

parser.add_option(

1416

'--pqm-submit-location', dest='pqm_submit_location', default=None,

1417

help=('The submit location for the pqm submit, if a pqm message is '

1418

'provided (see --submit-pqm-message). If this option is not '

1419

'provided, the script will look for an explicitly specified '

1420

'launchpad branch using the -b/--branch option; if that branch '

1421

'was specified and is owned by the launchpad-pqm user on '

1422

'launchpad, it is used as the pqm submit location. Otherwise, '

1423

'for local branches, bzr configuration is consulted; for '

1424

'remote branches, it is assumed that the submit branch is %s.'

1425

% (TRUNK_BRANCH,)))

1426

parser.add_option(

1427

'--pqm-email', dest='pqm_email', default=None,

1428

help=('Specify the email address of the PQM you are submitting to. '

1429

'If the branch is local, then the bzr configuration is '

1430

'consulted; for remote branches "Launchpad PQM '

1431

'<launchpad@pqm.canonical.com>" is used by default.'))

1432

parser.add_option(

1433

'-m', '--machine', dest='machine_id', default=None,

1434

help=('The AWS machine identifier (AMID) on which to base this run. '

1435

'You should typically only have to supply this if you are '

1436

'testing new AWS images. Defaults to trying to find the most '

1437

'recent one with an approved owner.'))

1438

parser.add_option(

1439

'-i', '--instance', dest='instance_type',

1440

default=DEFAULT_INSTANCE_TYPE,

1441

help=('The AWS instance type on which to base this run. '

1442

'Available options are %r. Defaults to `%s`.' %

1443

(AVAILABLE_INSTANCE_TYPES, DEFAULT_INSTANCE_TYPE)))

1444

parser.add_option(

1445

'-p', '--postmortem', dest='postmortem', default=False,

1446

action='store_true',

1447

help=('Drop to interactive prompt after the test and before shutting '

1448

'down the instance for postmortem analysis of the EC2 instance '

1449

'and/or of this script.'))

1450

parser.add_option(

1451

'--headless', dest='headless', default=False,

1452

action='store_true',

1453

help=('After building the instance and test, run the remote tests '

1454

'headless. Cannot be used with postmortem '

1455

'or file.'))

1456

parser.add_option(

1457

'-d', '--debug', dest='debug', default=False,

1458

action='store_true',

1459

help=('Drop to pdb trace as soon as possible.'))

1460

# Use tabs to force a newline in the help text.

1461

fake_newline = "\t\t\t\t\t\t\t"

1462

parser.add_option(

1463

'--demo', action='append', dest='demo_networks',

1464

help=("Don't run tests. Instead start a demo instance of Launchpad. "

1465

"You can allow multiple networks to access the demo by "

1466

"repeating the argument." + fake_newline +

1467

"Example: --demo 192.168.1.100 --demo 10.1.13.0/24" +

1468

fake_newline +

1469

"See" + fake_newline +

1470

"https://wiki.canonical.com/Launchpad/EC2Test/ForDemos" ))

1471

parser.add_option(

1472

'--open-browser', dest='open_browser', default=False,

1473

action='store_true',

1474

help=('Open the results page in your default browser'))

1475

parser.add_option(

1476

'-c', '--include-download-cache-changes',

1477

dest='include_download_cache_changes', action='store_true',

1478

help=('Include any changes in the download cache (added or unknown) '

1479

'in the download cache of the test run. Note that, if you have '

1480

'any changes in your download cache, trying to submit to pqm '

1481

'will always raise an error. Also note that, if you have any '

1482

'changes in your download cache, you must explicitly choose to '

1483

'include or ignore the changes.'))

1484

parser.add_option(

1485

'-g', '--ignore-download-cache-changes',

1486

dest='include_download_cache_changes', action='store_false',

1487

help=('Ignore any changes in the download cache (added or unknown) '

1488

'in the download cache of the test run. Note that, if you have '

1489

'any changes in your download cache, trying to submit to pqm '

1490

'will always raise an error. Also note that, if you have any '

1491

'changes in your download cache, you must explicitly choose to '

1492

'include or ignore the changes.'))

1493

options, args = parser.parse_args()

1494

if options.debug:

1495

import pdb; pdb.set_trace()

1496

if options.demo_networks:

1497

# We need the postmortem console to open the ec2 instance's

1498

# network access, and to keep the ec2 instance from being shutdown.

1499

options.postmortem = True

1500

if len(args) == 1:

1501

if options.trunk:

1502

parser.error(

1503

'Cannot supply both a branch and the --trunk argument.')

1504

branch = args[0]

1505

elif len(args) > 1:

1506

parser.error('Too many arguments.')

1507

elif options.trunk:

1508

branch = None

1509

else:

1510

branch = '.'

1511

if ((options.postmortem or options.file or options.demo_networks)

1512

and options.headless):

1513

parser.error(

1514

'Headless mode currently does not support postmortem, file '

1515

'or demo options.')

1516

if options.no_email:

1517

if options.email:

1518

parser.error(

1519

'May not supply both --no-email and an --email address')

1520

email = False

1521

else:

1522

email = options.email

1523

if email is None:

1524

email = True

1525

if options.instance_type not in AVAILABLE_INSTANCE_TYPES:

1526

parser.error('Unknown instance type.')

1527

if options.branches is None:

1528

branches = ()

1529

else:

1530

branches = [data.split('=', 1) for data in options.branches]

1531

runner = EC2TestRunner(

1532

branch, email=email, file=options.file,

1533

test_options=options.test_options, headless=options.headless,

1534

branches=branches,

1535

machine_id=options.machine_id, instance_type=options.instance_type,

1536

pqm_message=options.pqm_message,

1537

pqm_public_location=options.pqm_public_location,

1538

pqm_submit_location=options.pqm_submit_location,

1539

demo_networks=options.demo_networks,

1540

open_browser=options.open_browser, pqm_email=options.pqm_email,

1541

include_download_cache_changes=options.include_download_cache_changes,

1542

)

1543

e = None

1544

try:

1545

try:

1546

runner.start()

1547

runner.configure_system()

1548

runner.prepare_tests()

1549

if options.demo_networks:

1550

runner.start_demo_webserver()

1551

else:

1552

result = runner.run_tests()

1553

except Exception, e:

1554

# If we are running in demo or postmortem mode, it is really

1555

# helpful to see if there are any exceptions before it waits

1556

# in the console (in the finally block), and you can't figure

1557

# out why it's broken.

1558

traceback.print_exc()

1559

finally:

1560

try:

1561

# XXX: JonathanLange 2009-06-02: Blackbox alert! This gets at the

1562

# private _instance variable of runner. Instead, it should do

1563

# something smarter. For example, the demo networks stuff could be

1564

# extracted out to different, non-TestRunner class that has an

1565

# instance.

1566

if options.demo_networks and runner._instance is not None:

1567

demo_network_string = '\n'.join(

1568

' ' + network for network in options.demo_networks)

1569

# XXX: JonathanLange 2009-06-02: Blackbox alert! See above.

1570

ec2_ip = socket.gethostbyname(runner._instance.hostname)

1571

print (

1572

"\n\n"

1573

"********************** DEMO *************************\n"

1574

"It may take 20 seconds for the demo server to start up."

1575

"\nTo demo to other users, you still need to open up\n"

1576

"network access to the ec2 instance from their IPs by\n"

1577

"entering command like this in the interactive python\n"

1578

"interpreter at the end of the setup. "

1579

"\n runner.security_group.authorize("

1580

"'tcp', 443, 443, '10.0.0.5/32')\n\n"

1581

"These demo networks have already been granted access on "

1582

"port 80 and 443:\n" + demo_network_string +

1583

"\n\nYou also need to edit your /etc/hosts to point\n"

1584

"launchpad.dev at the ec2 instance's IP like this:\n"

1585

" " + ec2_ip + " launchpad.dev\n\n"

1586

"See "

1587

"<https://wiki.canonical.com/Launchpad/EC2Test/ForDemos>."

1588

"\n*****************************************************"

1589

"\n\n")

1590

# XXX: JonathanLange 2009-06-02: Blackbox alert! This uses the

1591

# private '_instance' variable and assumes that the runner has

1592

# exactly one instance.

1593

if options.postmortem and runner._instance is not None:

1594

console = code.InteractiveConsole({'runner': runner, 'e': e})

1595

console.interact((

1596

'Postmortem Console. EC2 instance is not yet dead.\n'

1597

'It will shut down when you exit this prompt (CTRL-D).\n'

1598

'\n'

1599

'Tab-completion is enabled.'

1600

'\n'

1601

'Test runner instance is available as `runner`.\n'

1602

'Also try these:\n'

1603

' http://%(dns)s/current_test.log\n'

1604

' ssh -A %(dns)s') %

1605

# XXX: JonathanLange 2009-06-02: Blackbox

1606

# alert! See above.

1607

{'dns': runner._instance.hostname})

1608

print 'Postmortem console closed.'

1609

finally:

1610

runner.shutdown()