~launchpad-pqm/launchpad/devel : revision 13693

10

from textwrap import dedent

11

12

from apt_pkg import TagFile

13

from testtools.matchers import StartsWith

13

from testtools.matchers import (

14

MatchesStructure,

15

StartsWith,

16

)

14

17

from zope.component import getUtility

15

18

16

19

from canonical.config import config

116

119

117

120

118

121

def get_marker_files(script, distroseries):

119

suites = [

120

distroseries.getSuite(pocket)

121

for pocket in pocketsuffix.iterkeys()]

122

return [script.locateIndexesMarker(suite) for suite in suites]

122

"""Return filesystem paths for all indexes markers for `distroseries`."""

123

suites = [

124

distroseries.getSuite(pocket) for pocket in pocketsuffix.iterkeys()]

125

distro = distroseries.distribution

126

return [script.locateIndexesMarker(distro, suite) for suite in suites]

123

127

124

128

125

129

class HelpersMixin:

289

293

self.SCRIPT_PATH + " -d ubuntu")

290

294

self.assertEqual(0, retval, "Script failure:\n" + stderr)

291

295

296

def test_getConfigs_maps_distro_and_purpose_to_matching_config(self):

297

distro = self.makeDistroWithPublishDirectory()

298

script = self.makeScript(distro)

299

script.setUp()

300

reference_config = getPubConfig(distro.main_archive)

301

config = script.getConfigs()[distro][ArchivePurpose.PRIMARY]

302

self.assertThat(

303

config, MatchesStructure.from_example(

304

reference_config, 'temproot', 'distroroot', 'archiveroot'))

305

306

def test_getConfigs_maps_distros(self):

307

distro = self.makeDistroWithPublishDirectory()

308

script = self.makeScript(distro)

309

script.setUp()

310

self.assertEqual([distro], script.getConfigs().keys())

311

292

312

def test_script_is_happy_with_no_publications(self):

293

313

distro = self.makeDistroWithPublishDirectory()

294

314

self.makeScript(distro).main()

366

386

367

387

def test_getDirtySuites_returns_suite_with_pending_publication(self):

368

388

spph = self.factory.makeSourcePackagePublishingHistory()

389

distro = spph.distroseries.distribution

369

390

script = self.makeScript(spph.distroseries.distribution)

370

391

script.setUp()

371

self.assertEqual([name_spph_suite(spph)], script.getDirtySuites())

392

self.assertEqual(

393

[name_spph_suite(spph)], script.getDirtySuites(distro))

372

394

373

395

def test_getDirtySuites_returns_suites_with_pending_publications(self):

374

396

distro = self.makeDistroWithPublishDirectory()

382

404

script.setUp()

383

405

self.assertContentEqual(

384

406

[name_spph_suite(spph) for spph in spphs],

385

script.getDirtySuites())

407

script.getDirtySuites(distro))

386

408

387

409

def test_getDirtySuites_ignores_suites_without_pending_publications(self):

388

410

spph = self.factory.makeSourcePackagePublishingHistory(

389

411

status=PackagePublishingStatus.PUBLISHED)

412

distro = spph.distroseries.distribution

390

413

script = self.makeScript(spph.distroseries.distribution)

391

414

script.setUp()

392

self.assertEqual([], script.getDirtySuites())

415

self.assertEqual([], script.getDirtySuites(distro))

393

416

394

417

def test_getDirtySecuritySuites_returns_security_suites(self):

395

418

distro = self.makeDistroWithPublishDirectory()

404

427

script.setUp()

405

428

self.assertContentEqual(

406

429

[name_spph_suite(spph) for spph in spphs],

407

script.getDirtySecuritySuites())

430

script.getDirtySecuritySuites(distro))

408

431

409

432

def test_getDirtySecuritySuites_ignores_non_security_suites(self):

410

433

distroseries = self.factory.makeDistroSeries()

419

442

distroseries=distroseries, pocket=pocket)

420

443

script = self.makeScript(distroseries.distribution)

421

444

script.setUp()

422

self.assertEqual([], script.getDirtySecuritySuites())

445

self.assertEqual(

446

[], script.getDirtySecuritySuites(distroseries.distribution))

423

447

424

448

def test_rsync_copies_files(self):

425

449

distro = self.makeDistroWithPublishDirectory()

431

455

os.makedirs(dists_backup)

432

456

os.makedirs(dists_root)

433

457

write_marker_file([dists_root, "new-file"], "New file")

434

script.rsyncBackupDists()

458

script.rsyncBackupDists(distro)

435

459

self.assertEqual(

436

460

"New file", read_marker_file([dists_backup, "new-file"]))

437

461

445

469

old_file = [dists_backup, "old-file"]

446

470

write_marker_file(old_file, "old-file")

447

471

os.makedirs(get_dists_root(get_pub_config(distro)))

448

script.rsyncBackupDists()

472

script.rsyncBackupDists(distro)

449

473

self.assertFalse(path_exists(*old_file))

450

474

451

475

def test_setUpDirs_creates_directory_structure(self):

478

502

script.setUp()

479

503

script.setUpDirs()

480

504

script.runParts = FakeMethod()

481

script.publishDistroArchive(distro.main_archive)

505

script.publishDistroArchive(distro, distro.main_archive)

482

506

self.assertEqual(1, script.runParts.call_count)

483

507

args, kwargs = script.runParts.calls[0]

484

parts_dir, env = args

508

run_distro, parts_dir, env = args

509

self.assertEqual(distro, run_distro)

485

510

self.assertEqual("publish-distro.d", parts_dir)

486

511

487

512

def test_runPublishDistroParts_passes_parameters(self):

490

515

script.setUp()

491

516

script.setUpDirs()

492

517

script.runParts = FakeMethod()

493

script.runPublishDistroParts(distro.main_archive)

518

script.runPublishDistroParts(distro, distro.main_archive)

494

519

args, kwargs = script.runParts.calls[0]

495

parts_dir, env = args

520

run_distro, parts_dir, env = args

496

521

required_parameters = set([

497

522

"ARCHIVEROOT", "DISTSROOT", "OVERRIDEROOT"])

498

523

missing_parameters = required_parameters.difference(set(env.keys()))

503

528

script = self.makeScript(distro)

504

529

script.setUp()

505

530

script.setUpDirs()

506

script.generateListings()

531

script.generateListings(distro)

507

532

pass

508

533

509

534

def test_clearEmptyDirs_cleans_up_empty_directories(self):

514

539

empty_dir = os.path.join(

515

540

get_dists_root(get_pub_config(distro)), 'empty-dir')

516

541

os.makedirs(empty_dir)

517

script.clearEmptyDirs()

542

script.clearEmptyDirs(distro)

518

543

self.assertFalse(file_exists(empty_dir))

519

544

520

545

def test_clearEmptyDirs_does_not_clean_up_nonempty_directories(self):

526

551

get_dists_root(get_pub_config(distro)), 'nonempty-dir')

527

552

os.makedirs(nonempty_dir)

528

553

write_marker_file([nonempty_dir, "placeholder"], "Data here!")

529

script.clearEmptyDirs()

554

script.clearEmptyDirs(distro)

530

555

self.assertTrue(file_exists(nonempty_dir))

531

556

532

557

def test_processOptions_finds_distribution(self):

534

559

script = self.makeScript(distro)

535

560

script.processOptions()

536

561

self.assertEqual(distro.name, script.options.distribution)

537

self.assertEqual(distro, script.distribution)

562

self.assertEqual([distro], script.distributions)

563

564

def test_processOptions_for_all_derived_finds_derived_distros(self):

565

dsp = self.factory.makeDistroSeriesParent()

566

script = PublishFTPMaster(test_args=['--all-derived'])

567

script.processOptions()

568

self.assertIn(dsp.derived_series.distribution, script.distributions)

569

570

def test_processOptions_for_all_derived_ignores_nonderived_distros(self):

571

distro = self.factory.makeDistribution()

572

script = PublishFTPMaster(test_args=['--all-derived'])

573

script.processOptions()

574

self.assertNotIn(distro, script.distributions)

538

575

539

576

def test_processOptions_complains_about_unknown_distribution(self):

540

577

script = self.makeScript()

545

582

self.enableRunParts()

546

583

script = self.makeScript(self.prepareUbuntu())

547

584

script.setUp()

585

distro = script.distributions[0]

548

586

script.executeShell = FakeMethod()

549

script.runParts("finalize.d", {})

587

script.runParts(distro, "finalize.d", {})

550

588

self.assertEqual(1, script.executeShell.call_count)

551

589

args, kwargs = script.executeShell.calls[-1]

552

590

command_line, = args

559

597

self.enableRunParts()

560

598

script = self.makeScript(self.prepareUbuntu())

561

599

script.setUp()

600

distro = script.distributions[0]

562

601

script.executeShell = FakeMethod()

563

602

key = self.factory.getUniqueString()

564

603

value = self.factory.getUniqueString()

565

script.runParts("finalize.d", {key: value})

604

script.runParts(distro, "finalize.d", {key: value})

566

605

args, kwargs = script.executeShell.calls[-1]

567

606

command_line, = args

568

607

self.assertIn("%s=%s" % (key, value), command_line)

595

634

def test_runFinalizeParts_passes_parameters(self):

596

635

script = self.makeScript(self.prepareUbuntu())

597

636

script.setUp()

637

distro = script.distributions[0]

598

638

script.runParts = FakeMethod()

599

script.runFinalizeParts()

639

script.runFinalizeParts(distro)

600

640

args, kwargs = script.runParts.calls[0]

601

parts_dir, env = args

641

run_distro, parts_dir, env = args

602

642

required_parameters = set(["ARCHIVEROOTS", "SECURITY_UPLOAD_ONLY"])

603

643

missing_parameters = required_parameters.difference(set(env.keys()))

604

644

self.assertEqual(set(), missing_parameters)

606

646

def test_publishSecurityUploads_skips_pub_if_no_security_updates(self):

607

647

script = self.makeScript()

608

648

script.setUp()

649

distro = script.distributions[0]

609

650

script.setUpDirs()

610

651

script.installDists = FakeMethod()

611

script.publishSecurityUploads()

652

script.publishSecurityUploads(distro)

612

653

self.assertEqual(0, script.installDists.call_count)

613

654

614

def test_publishAllUploads_publishes_all_distro_archives(self):

655

def test_publishDistroUploads_publishes_all_distro_archives(self):

615

656

distro = self.makeDistroWithPublishDirectory()

616

657

distroseries = self.factory.makeDistroSeries(distribution=distro)

617

658

partner_archive = self.factory.makeArchive(

624

665

script.setUp()

625

666

script.setUpDirs()

626

667

script.publishDistroArchive = FakeMethod()

627

script.publishAllUploads()

668

script.publishDistroUploads(distro)

628

669

published_archives = [

629

args[0] for args, kwargs in script.publishDistroArchive.calls]

670

args[1] for args, kwargs in script.publishDistroArchive.calls]

630

671

631

672

self.assertContentEqual(

632

673

distro.all_distro_archives, published_archives)

648

689

script.logger = BufferLogger()

649

690

script.logger.setLevel(logging.INFO)

650

691

script.setUpDirs()

651

archive_config = script.configs[ArchivePurpose.PRIMARY]

692

archive_config = getPubConfig(distro.main_archive)

652

693

backup_dists = os.path.join(

653

694

archive_config.archiveroot + "-distscopy", "dists")

654

695

working_dists = get_working_dists(archive_config)

677

718

args, kwargs = script.publish.calls[0]

678

719

self.assertEqual({'security_only': True}, kwargs)

679

720

680

def test_publishAllUploads_processes_all_archives(self):

721

def test_publishDistroUploads_processes_all_archives(self):

681

722

distro = self.makeDistroWithPublishDirectory()

682

723

partner_archive = self.factory.makeArchive(

683

724

distribution=distro, purpose=ArchivePurpose.PARTNER)

684

725

script = self.makeScript(distro)

685

726

script.publishDistroArchive = FakeMethod()

686

727

script.setUp()

687

script.publishAllUploads()

728

script.publishDistroUploads(distro)

688

729

published_archives = [

689

args[0] for args, kwargs in script.publishDistroArchive.calls]

730

args[1] for args, kwargs in script.publishDistroArchive.calls]

690

731

self.assertContentEqual(

691

732

[distro.main_archive, partner_archive], published_archives)

692

733

726

767

done

727

768

"""))

728

769

729

script.runFinalizeParts()

770

script.runFinalizeParts(distro)

730

771

731

772

for archive in [distro.main_archive, distro.getArchive("partner")]:

732

773

archive_root = getPubConfig(archive).archiveroot

745

786

746

787

message = self.factory.getUniqueString()

747

788

script = self.makeScript()

748

script.publishAllUploads = FakeMethod(failure=MoonPhaseError(message))

789

script.publishDistroUploads = FakeMethod(

790

failure=MoonPhaseError(message))

749

791

script.setUp()

750

self.assertRaisesWithContent(MoonPhaseError, message, script.publish)

792

self.assertRaisesWithContent(

793

MoonPhaseError, message,

794

script.publish, script.distributions[0])

751

795

752

796

def test_publish_obeys_keyboard_interrupt(self):

753

797

# Similar to an Exception, a keyboard interrupt does not get

754

798

# swallowed.

755

799

message = self.factory.getUniqueString()

756

800

script = self.makeScript()

757

script.publishAllUploads = FakeMethod(

801

script.publishDistroUploads = FakeMethod(

758

802

failure=KeyboardInterrupt(message))

759

803

script.setUp()

760

804

self.assertRaisesWithContent(

761

KeyboardInterrupt, message, script.publish)

805

KeyboardInterrupt, message,

806

script.publish, script.distributions[0])

762

807

763

808

def test_publish_recovers_working_dists_on_exception(self):

764

809

# If an Exception comes up while publishing, the publish method

769

814

failure = MoonPhaseError(self.factory.getUniqueString())

770

815

771

816

script = self.makeScript()

772

script.publishAllUploads = FakeMethod(failure=failure)

817

script.publishDistroUploads = FakeMethod(failure=failure)

773

818

script.recoverArchiveWorkingDir = FakeMethod()

774

819

script.setUp()

775

820

776

821

try:

777

script.publish()

822

script.publish(script.distributions[0])

778

823

except MoonPhaseError:

779

824

pass

780

825

786

831

failure = KeyboardInterrupt("Ctrl-C!")

787

832

788

833

script = self.makeScript()

789

script.publishAllUploads = FakeMethod(failure=failure)

834

script.publishDistroUploads = FakeMethod(failure=failure)

790

835

script.recoverArchiveWorkingDir = FakeMethod()

791

836

script.setUp()

792

837

793

838

try:

794

script.publish()

839

script.publish(script.distributions[0])

795

840

except KeyboardInterrupt:

796

841

pass

797

842

804

849

805

850

def createIndexesMarkerDir(self, script, distroseries):

806

851

"""Create the directory for `distroseries`'s indexes marker."""

807

marker = script.locateIndexesMarker(get_a_suite(distroseries))

852

marker = script.locateIndexesMarker(

853

distroseries.distribution, get_a_suite(distroseries))

808

854

os.makedirs(os.path.dirname(marker))

809

855

810

856

def makeDistroSeriesNeedingIndexes(self, distribution=None):

860

906

861

907

needful_suites = script.listSuitesNeedingIndexes(series)

862

908

suite = get_a_suite(series)

863

script.markIndexCreationComplete(suite)

909

script.markIndexCreationComplete(distro, suite)

864

910

needful_suites.remove(suite)

865

911

self.assertContentEqual(

866

912

needful_suites, script.listSuitesNeedingIndexes(series))

885

931

script.markIndexCreationComplete = FakeMethod()

886

932

script.runPublishDistro = FakeMethod()

887

933

suite = get_a_suite(series)

888

script.createIndexes(suite)

934

script.createIndexes(distro, suite)

889

935

self.assertEqual(

890

[((suite, ), {})], script.markIndexCreationComplete.calls)

936

[((distro, suite), {})], script.markIndexCreationComplete.calls)

891

937

892

938

def test_failed_index_creation_is_not_marked_complete(self):

893

939

# If index creation fails, it is not marked as having been

900

946

script.markIndexCreationComplete = FakeMethod()

901

947

script.runPublishDistro = FakeMethod(failure=Boom("Sorry!"))

902

948

try:

903

script.createIndexes(get_a_suite(series))

949

script.createIndexes(series.distribution, get_a_suite(series))

904

950

except:

905

951

pass

906

952

self.assertEqual([], script.markIndexCreationComplete.calls)

911

957

series = self.factory.makeDistroSeries()

912

958

script = self.makeScript(series.distribution)

913

959

script.setUp()

914

archive_root = script.configs[ArchivePurpose.PRIMARY].archiveroot

960

archive_root = getPubConfig(series.main_archive).archiveroot

915

961

self.assertThat(

916

script.locateIndexesMarker(get_a_suite(series)),

962

script.locateIndexesMarker(

963

series.distribution, get_a_suite(series)),

917

964

StartsWith(os.path.normpath(archive_root)))

918

965

919

966

def test_locateIndexesMarker_uses_separate_files_per_suite(self):

946

993

script.setUp()

947

994

suite = get_a_suite(series)

948

995

self.assertThat(

949

os.path.basename(script.locateIndexesMarker(suite)),

996

os.path.basename(script.locateIndexesMarker(

997

series.distribution, suite)),

950

998

StartsWith("."))

951

999

952

1000

def test_script_calls_createIndexes_for_new_series(self):

958

1006

script.createIndexes = FakeMethod()

959

1007

script.main()

960

1008

expected_calls = [

961

((series.getSuite(pocket), ), {})

1009

((distro, series.getSuite(pocket)), {})

962

1010

for pocket in pocketsuffix.iterkeys()]

963

1011

self.assertContentEqual(expected_calls, script.createIndexes.calls)

964

1012

974

1022

self.createIndexesMarkerDir(script, series)

975

1023

suite = get_a_suite(series)

976

1024

977

script.createIndexes(suite)

1025

script.createIndexes(distro, suite)

978

1026

979

1027

args, kwargs = script.runPublishDistro.calls[0]

980

1028

self.assertEqual([suite], kwargs['suites'])

994

1042

script.main()

995

1043

self.assertEqual([], script.listSuitesNeedingIndexes(series))

996

1044

sources = os.path.join(

997

script.configs[ArchivePurpose.PRIMARY].distsroot,

1045

getPubConfig(series.main_archive).distsroot,

998

1046

series.name, "main", "source", "Sources")

999

1047

self.assertTrue(file_exists(sources))