~launchpad-pqm/launchpad/devel

# Copyright 2011 Canonical Ltd.  This software is licensed under the

# GNU Affero General Public License version 3 (see the file LICENSE).

"""Tests for `lp.registry.browser.distroseries`."""

__metaclass__ = type

from datetime import timedelta

import difflib

import re

from textwrap import TextWrapper

from BeautifulSoup import BeautifulSoup

from lazr.restful.interfaces import IJSONRequestCache

from lxml import html

import soupmatchers

from storm.zope.interfaces import IResultSet

from testtools.content import (

    Content,

    text_content,

    )

from testtools.content_type import UTF8_TEXT

from testtools.matchers import (

    EndsWith,

    Equals,

    LessThan,

    Not,

    )

from zope.component import getUtility

from zope.security.proxy import removeSecurityProxy

from canonical.config import config

from canonical.database.constants import UTC_NOW

from canonical.database.sqlbase import flush_database_caches

from canonical.launchpad.testing.pages import (

    extract_text,

    find_tag_by_id,

    )

from canonical.launchpad.webapp.authorization import check_permission

from canonical.launchpad.webapp.batching import BatchNavigator

from canonical.launchpad.webapp.interaction import get_current_principal

from canonical.launchpad.webapp.interfaces import BrowserNotificationLevel

from canonical.launchpad.webapp.publisher import canonical_url

from canonical.testing.layers import (

    DatabaseFunctionalLayer,

    LaunchpadFunctionalLayer,

    LaunchpadZopelessLayer,

    )

from lp.app.interfaces.launchpad import ILaunchpadCelebrities

from lp.archivepublisher.debversion import Version

from lp.registry.browser.distroseries import (

    HIGHER_VERSION_THAN_PARENT,

    ALL,

    NON_IGNORED,

    RESOLVED,

    seriesToVocab,

    )

from lp.registry.enum import (

    DistroSeriesDifferenceStatus,

    DistroSeriesDifferenceType,

    )

from lp.registry.interfaces.pocket import PackagePublishingPocket

from lp.registry.interfaces.series import SeriesStatus

from lp.services.features import (

    get_relevant_feature_controller,

    getFeatureFlag,

    )

from lp.services.features.testing import FeatureFixture

from lp.services.utils import utc_now

from lp.soyuz.enums import (

    ArchivePermissionType,

    PackagePublishingStatus,

    SourcePackageFormat,

    )

from lp.soyuz.interfaces.component import IComponentSet

from lp.soyuz.interfaces.distributionjob import (

    IDistroSeriesDifferenceJobSource,

    IInitializeDistroSeriesJobSource,

    )

from lp.soyuz.interfaces.packagecopyjob import IPlainPackageCopyJobSource

from lp.soyuz.interfaces.sourcepackageformat import (

    ISourcePackageFormatSelectionSet,

    )

from lp.soyuz.model import distroseriesdifferencejob

from lp.soyuz.model.archivepermission import ArchivePermission

from lp.soyuz.model.packagecopyjob import PlainPackageCopyJob

from lp.testing import (

    ANONYMOUS,

    anonymous_logged_in,

    celebrity_logged_in,

    login,

    login_celebrity,

    login_person,

    normalize_whitespace,

    person_logged_in,

    StormStatementRecorder,

    TestCaseWithFactory,

    with_celebrity_logged_in,

    )

from lp.testing.fakemethod import FakeMethod

from lp.testing.matchers import (

    DocTestMatches,

    EqualsIgnoringWhitespace,

    HasQueryCount,

    )

from lp.testing.views import create_initialized_view

def set_derived_series_ui_feature_flag(test_case):

    test_case.useFixture(FeatureFixture({

        u'soyuz.derived_series_ui.enabled': u'on',

        }))

def set_derived_series_sync_feature_flag(test_case):

    test_case.useFixture(FeatureFixture({

        u'soyuz.derived_series_sync.enabled': u'on',

        u'soyuz.derived_series_ui.enabled': u'on',

        }))

def set_derived_series_difference_jobs_feature_flag(test_case):

    test_case.useFixture(FeatureFixture({

        distroseriesdifferencejob.FEATURE_FLAG_ENABLE_MODULE: u'on',

        }))

class TestDistroSeriesView(TestCaseWithFactory):

    """Test the distroseries +index view."""

    layer = LaunchpadZopelessLayer

    def test_needs_linking(self):

        ubuntu = getUtility(ILaunchpadCelebrities).ubuntu

        distroseries = self.factory.makeDistroSeries(distribution=ubuntu)

        view = create_initialized_view(distroseries, '+index')

        self.assertEqual(view.needs_linking, None)

    def _createDifferenceAndGetView(self, difference_type):

        # Helper function to create a valid DSD.

        dsp = self.factory.makeDistroSeriesParent()

        self.factory.makeDistroSeriesDifference(

            derived_series=dsp.derived_series,

            difference_type=difference_type)

        return create_initialized_view(dsp.derived_series, '+index')

    def test_num_differences(self):

        diff_type = DistroSeriesDifferenceType.DIFFERENT_VERSIONS

        view = self._createDifferenceAndGetView(diff_type)

        self.assertEqual(1, view.num_differences)

    def test_num_differences_in_parent(self):

        diff_type = DistroSeriesDifferenceType.MISSING_FROM_DERIVED_SERIES

        view = self._createDifferenceAndGetView(diff_type)

        self.assertEqual(1, view.num_differences_in_parent)

    def test_num_differences_in_child(self):

        diff_type = DistroSeriesDifferenceType.UNIQUE_TO_DERIVED_SERIES

        view = self._createDifferenceAndGetView(diff_type)

        self.assertEqual(1, view.num_differences_in_child)

class DistroSeriesIndexFunctionalTestCase(TestCaseWithFactory):

    """Test the distroseries +index page."""

    layer = DatabaseFunctionalLayer

    def _setupDifferences(self, name, parent_names, nb_diff_versions,

                          nb_diff_child, nb_diff_parent):

        # Helper to create DSDs of the different types.

        derived_series = self.factory.makeDistroSeries(name=name)

        self.simple_user = self.factory.makePerson()

        # parent_names can be a list of parent names or a single name

        # for a single parent (e.g. ['parent1_name', 'parent2_name'] or

        # 'parent_name').

        # If multiple parents are created, the DSDs will be created with

        # the first one.

        if type(parent_names) == str:

            parent_names = [parent_names]

        dsps = []

        for parent_name in parent_names:

            parent_series = self.factory.makeDistroSeries(name=parent_name)

            dsps.append(self.factory.makeDistroSeriesParent(

                derived_series=derived_series, parent_series=parent_series))

        first_parent_series = dsps[0].parent_series

        for i in range(nb_diff_versions):

            diff_type = DistroSeriesDifferenceType.DIFFERENT_VERSIONS

            self.factory.makeDistroSeriesDifference(

                derived_series=derived_series,

                difference_type=diff_type,

                parent_series=first_parent_series)

        for i in range(nb_diff_child):

            diff_type = DistroSeriesDifferenceType.MISSING_FROM_DERIVED_SERIES

            self.factory.makeDistroSeriesDifference(

                derived_series=derived_series,

                difference_type=diff_type,

                parent_series=first_parent_series)

        for i in range(nb_diff_parent):

            diff_type = DistroSeriesDifferenceType.UNIQUE_TO_DERIVED_SERIES

            self.factory.makeDistroSeriesDifference(

                derived_series=derived_series,

                difference_type=diff_type,

                parent_series=first_parent_series)

        return derived_series

    def test_differences_no_flag_no_portlet(self):

        # The portlet is not displayed if the feature flag is not enabled.

        derived_series = self._setupDifferences('deri', 'sid', 1, 2, 2)

        portlet_header = soupmatchers.HTMLContains(

            soupmatchers.Tag(

                'Derivation portlet header', 'h2',

                text='Derived from Sid'),

            )

        with person_logged_in(self.simple_user):

            view = create_initialized_view(

                derived_series,

                '+index',

                principal=self.simple_user)

            html_content = view()

        self.assertEqual(

            None, getFeatureFlag('soyuz.derived_series_ui.enabled'))

        self.assertThat(html_content, Not(portlet_header))

    def test_differences_portlet_all_differences(self):

        # The difference portlet shows the differences with the parent

        # series.

        set_derived_series_ui_feature_flag(self)

        derived_series = self._setupDifferences('deri', 'sid', 1, 2, 3)

        portlet_display = soupmatchers.HTMLContains(

            soupmatchers.Tag(

                'Derivation portlet header', 'h2',

                text='Derived from Sid'),

            soupmatchers.Tag(

                'Differences link', 'a',

                text=re.compile('\s*1 package with differences\s*'),

                attrs={'href': re.compile('.*/\+localpackagediffs')}),

            soupmatchers.Tag(

                'Parent diffs link', 'a',

                text=re.compile('\s*2 packages only in Sid\s*'),

                attrs={'href': re.compile('.*/\+missingpackages')}),

            soupmatchers.Tag(

                'Child diffs link', 'a',

                text=re.compile('\s*3 packages only in Deri\s*'),

                attrs={'href': re.compile('.*/\+uniquepackages')}))

        with person_logged_in(self.simple_user):

            view = create_initialized_view(

                derived_series,

                '+index',

                principal=self.simple_user)

            # XXX rvb 2011-04-12 bug=758649: LaunchpadTestRequest overrides

            # self.features to NullFeatureController.

            view.request.features = get_relevant_feature_controller()

            html_content = view()

        self.assertThat(html_content, portlet_display)

    def test_differences_portlet_all_differences_multiple_parents(self):

        # The difference portlet shows the differences with the multiple

        # parent series.

        set_derived_series_ui_feature_flag(self)

        derived_series = self._setupDifferences(

            'deri', ['sid1', 'sid2'], 0, 1, 0)

        portlet_display = soupmatchers.HTMLContains(

            soupmatchers.Tag(

                'Derivation portlet header', 'h2',

                text='Derived from 2 parents'),

            soupmatchers.Tag(

                'Parent diffs link', 'a',

                text=re.compile('\s*1 package only in a parent series\s*'),

                attrs={'href': re.compile('.*/\+missingpackages')}))

        with person_logged_in(self.simple_user):

            view = create_initialized_view(

                derived_series,

                '+index',

                principal=self.simple_user)

            # XXX rvb 2011-04-12 bug=758649: LaunchpadTestRequest overrides

            # self.features to NullFeatureController.

            view.request.features = get_relevant_feature_controller()

            html_text = view()

        self.assertThat(html_text, portlet_display)

    def test_differences_portlet_no_differences(self):

        # The difference portlet displays 'No differences' if there is no

        # differences with the parent.

        set_derived_series_ui_feature_flag(self)

        derived_series = self._setupDifferences('deri', 'sid', 0, 0, 0)

        portlet_display = soupmatchers.HTMLContains(

            soupmatchers.Tag(

                'Derivation portlet header', 'h2',

                text='Derived from Sid'),

            soupmatchers.Tag(

                'Child diffs link', True,

                text=re.compile('\s*No differences\s*')),

              )

        with person_logged_in(self.simple_user):

            view = create_initialized_view(

                derived_series,

                '+index',

                principal=self.simple_user)

            # XXX rvb 2011-04-12 bug=758649: LaunchpadTestRequest overrides

            # self.features to NullFeatureController.

            view.request.features = get_relevant_feature_controller()

            html_content = view()

        self.assertThat(html_content, portlet_display)

    def test_differences_portlet_initializing(self):

        # The difference portlet displays 'The series is initializing.' if

        # there is an initializing job for the series.

        set_derived_series_ui_feature_flag(self)

        derived_series = self.factory.makeDistroSeries()

        parent_series = self.factory.makeDistroSeries()

        self.simple_user = self.factory.makePerson()

        job_source = getUtility(IInitializeDistroSeriesJobSource)

        job_source.create(derived_series, [parent_series.id])

        portlet_display = soupmatchers.HTMLContains(

            soupmatchers.Tag(

                'Derived series', 'h2',

                text='Series initialization in progress'),

            soupmatchers.Tag(

                'Init message', True,

                text=re.compile('\s*This series is initializing.\s*')),

              )

        with person_logged_in(self.simple_user):

            view = create_initialized_view(

                derived_series,

                '+index',

                principal=self.simple_user)

            # XXX rvb 2011-04-12 bug=758649: LaunchpadTestRequest overrides

            # self.features to NullFeatureController.

            view.request.features = get_relevant_feature_controller()

            html_content = view()

        self.assertTrue(derived_series.isInitializing())

        self.assertThat(html_content, portlet_display)

    def test_differences_portlet_initialization_failed(self):

        # The difference portlet displays a failure message if initialization

        # for the series has failed.

        set_derived_series_ui_feature_flag(self)

        derived_series = self.factory.makeDistroSeries()

        parent_series = self.factory.makeDistroSeries()

        self.simple_user = self.factory.makePerson()

        job_source = getUtility(IInitializeDistroSeriesJobSource)

        job = job_source.create(derived_series, [parent_series.id])

        job.start()

        job.fail()

        portlet_display = soupmatchers.HTMLContains(

            soupmatchers.Tag(

                'Derived series', 'h2',

                text='Series initialization has failed'),

            )

        with person_logged_in(self.simple_user):

            view = create_initialized_view(

                derived_series, '+index', principal=self.simple_user)

            html_content = view()

        self.assertThat(html_content, portlet_display)

    def assertInitSeriesLinkPresent(self, series, person):

        self._assertInitSeriesLink(series, person, True)

    def assertInitSeriesLinkNotPresent(self, series, person):

        self._assertInitSeriesLink(series, person, False)

    def _assertInitSeriesLink(self, series, person, present=True):

        # Helper method to check the presence/absence of the link to

        # +initseries.

        if person == 'admin':

            person = getUtility(ILaunchpadCelebrities).admin.teamowner

        init_link_matcher = soupmatchers.HTMLContains(

            soupmatchers.Tag(

                'Initialize series', 'a',

                text='Initialize series',

                attrs={'href': '%s/+initseries' % canonical_url(series)}))

        with person_logged_in(person):

            view = create_initialized_view(

                series,

                '+index',

                principal=person)

            html_content = view()

        if present:

            self.assertThat(html_content, init_link_matcher)

        else:

            self.assertThat(html_content, Not(init_link_matcher))

    def test_differences_init_link_no_feature(self):

        # The link to +initseries is not displayed if the feature flag

        # is not enabled.

        series = self.factory.makeDistroSeries()

        self.assertInitSeriesLinkNotPresent(series, 'admin')

    def test_differences_init_link_admin(self):

        # The link to +initseries is displayed to admin users if the

        # feature flag is enabled.

        set_derived_series_ui_feature_flag(self)

        series = self.factory.makeDistroSeries()

        self.assertInitSeriesLinkPresent(series, 'admin')

    def test_differences_init_link_series_driver(self):

        # The link to +initseries is displayed to the distroseries's

        # drivers.

        set_derived_series_ui_feature_flag(self)

        distroseries = self.factory.makeDistroSeries()

        driver = self.factory.makePerson()

        with celebrity_logged_in('admin'):

            distroseries.driver = driver

        self.assertInitSeriesLinkPresent(distroseries, driver)

    def test_differences_init_link_not_admin(self):

        # The link to +initseries is not displayed to not admin users if the

        # feature flag is enabled.

        set_derived_series_ui_feature_flag(self)

        series = self.factory.makeDistroSeries()

        person = self.factory.makePerson()

        self.assertInitSeriesLinkNotPresent(series, person)

    def test_differences_init_link_initialized(self):

        # The link to +initseries is not displayed if the series is

        # already initialized (i.e. has any published package).

        set_derived_series_ui_feature_flag(self)

        series = self.factory.makeDistroSeries()

        self.factory.makeSourcePackagePublishingHistory(

            archive=series.main_archive,

            distroseries=series)

        self.assertInitSeriesLinkNotPresent(series, 'admin')

    def test_differences_init_link_series_initializing(self):

        # The link to +initseries is not displayed if the series is

        # initializing.

        set_derived_series_ui_feature_flag(self)

        series = self.factory.makeDistroSeries()

        parent_series = self.factory.makeDistroSeries()

        job_source = getUtility(IInitializeDistroSeriesJobSource)

        job_source.create(series, [parent_series.id])

        self.assertInitSeriesLinkNotPresent(series, 'admin')

class TestDistroSeriesDerivationPortlet(TestCaseWithFactory):

    layer = LaunchpadFunctionalLayer

    @property

    def job_source(self):

        return getUtility(IInitializeDistroSeriesJobSource)

    def test_initialization_failed_can_retry(self):

        # When initialization has failed and the user has the ability to retry

        # it prompts the user to try again.

        set_derived_series_ui_feature_flag(self)

        series = self.factory.makeDistroSeries()

        parent = self.factory.makeDistroSeries()

        job = self.job_source.create(series, [parent.id])

        job.start()

        job.fail()

        with person_logged_in(series.owner):

            view = create_initialized_view(series, '+portlet-derivation')

            html_content = view()

        self.assertThat(

            extract_text(html_content), DocTestMatches(

                "Series initialization has failed\n"

                "You can attempt initialization again."))

    def test_initialization_failed_cannot_retry(self):

        # When initialization has failed and the user does not have the

        # ability to retry it suggests contacting someone who can.

        set_derived_series_ui_feature_flag(self)

        series = self.factory.makeDistroSeries()

        parent = self.factory.makeDistroSeries()

        job = self.job_source.create(series, [parent.id])

        job.start()

        job.fail()

        with person_logged_in(series.distribution.owner):

            series.distribution.owner.displayname = u"Bob Individual"

        with anonymous_logged_in():

            view = create_initialized_view(series, '+portlet-derivation')

            html_content = view()

        self.assertThat(

            extract_text(html_content), DocTestMatches(

                "Series initialization has failed\n"

                "You cannot attempt initialization again, "

                "but Bob Individual may be able to help."))

        # When the owner is a team the message differs slightly from when the

        # owner is an individual.

        with person_logged_in(series.distribution.owner):

            series.distribution.owner = self.factory.makeTeam(

                displayname=u"Team Teamy Team Team")

        with anonymous_logged_in():

            view = create_initialized_view(series, '+portlet-derivation')

            html_content = view()

        self.assertThat(

            extract_text(html_content), DocTestMatches(

                "Series initialization has failed\n"

                "You cannot attempt initialization again, but a "

                "member of Team Teamy Team Team may be able to help."))

class TestMilestoneBatchNavigatorAttribute(TestCaseWithFactory):

    """Test the series.milestone_batch_navigator attribute."""

    layer = LaunchpadZopelessLayer

    def test_distroseries_milestone_batch_navigator(self):

        ubuntu = getUtility(ILaunchpadCelebrities).ubuntu

        distroseries = self.factory.makeDistroSeries(distribution=ubuntu)

        for name in ('a', 'b', 'c', 'd'):

            distroseries.newMilestone(name)

        view = create_initialized_view(distroseries, name='+index')

        self._check_milestone_batch_navigator(view)

    def test_productseries_milestone_batch_navigator(self):

        product = self.factory.makeProduct()

        for name in ('a', 'b', 'c', 'd'):

            product.development_focus.newMilestone(name)

        view = create_initialized_view(

            product.development_focus, name='+index')

        self._check_milestone_batch_navigator(view)

    def _check_milestone_batch_navigator(self, view):

        config.push('default-batch-size', """

        [launchpad]

        default_batch_size: 2

        """)

        self.assert_(

            isinstance(view.milestone_batch_navigator, BatchNavigator),

            'milestone_batch_navigator is not a BatchNavigator object: %r'

            % view.milestone_batch_navigator)

        self.assertEqual(4, view.milestone_batch_navigator.batch.total())

        expected = [

            'd',

            'c',

            ]

        milestone_names = [

            item.name

            for item in view.milestone_batch_navigator.currentBatch()]

        self.assertEqual(expected, milestone_names)

        config.pop('default-batch-size')

class TestDistroSeriesAddView(TestCaseWithFactory):

    layer = DatabaseFunctionalLayer

    def setUp(self):

        super(TestDistroSeriesAddView, self).setUp()

        self.user = self.factory.makePerson()

        self.distribution = self.factory.makeDistribution(owner=self.user)

    def createNewDistroseries(self):

        form = {

            "field.name": u"polished",

            "field.version": u"12.04",

            "field.displayname": u"Polished Polecat",

            "field.summary": u"Even The Register likes it.",

            "field.actions.create": u"Add Series",

            }

        with person_logged_in(self.user):

            create_initialized_view(self.distribution, "+addseries",

                                    form=form)

        distroseries = self.distribution.getSeries(u"polished")

        return distroseries

    def assertCreated(self, distroseries):

        self.assertEqual(u"polished", distroseries.name)

        self.assertEqual(u"12.04", distroseries.version)

        self.assertEqual(u"Polished Polecat", distroseries.displayname)

        self.assertEqual(u"Polished Polecat", distroseries.title)

        self.assertEqual(u"Even The Register likes it.", distroseries.summary)

        self.assertEqual(u"", distroseries.description)

        self.assertEqual(self.user, distroseries.owner)

    def test_plain_submit(self):

        # When creating a new DistroSeries via DistroSeriesAddView, the title

        # is set to the same as the displayname (title is, in any case,

        # deprecated), the description is left empty, and previous_series is

        # None (DistroSeriesInitializeView takes care of setting that).

        distroseries = self.createNewDistroseries()

        self.assertCreated(distroseries)

        self.assertIs(None, distroseries.previous_series)

    def test_submit_sets_previous_series(self):

        # Creating a new series when one already exists should set the

        # previous_series.

        old_series = self.factory.makeDistroSeries(

            self.distribution, version='11.10')

        # A yet older series.

        self.factory.makeDistroSeries(

            self.distribution, version='11.04')

        old_time = utc_now() - timedelta(days=5)

        removeSecurityProxy(old_series).datereleased = old_time

        distroseries = self.createNewDistroseries()

        self.assertEqual(old_series, distroseries.previous_series)

class TestDistroSeriesInitializeView(TestCaseWithFactory):

    layer = DatabaseFunctionalLayer

    def test_init(self):

        # There exists a +initseries view for distroseries.

        distroseries = self.factory.makeDistroSeries()

        view = create_initialized_view(distroseries, "+initseries")

        self.assertTrue(view)

    def test_is_derived_series_feature_enabled(self):

        # The feature is disabled by default, but can be enabled by setting

        # the soyuz.derived_series_ui.enabled flag.

        distroseries = self.factory.makeDistroSeries()

        view = create_initialized_view(distroseries, "+initseries")

        with FeatureFixture({}):

            self.assertFalse(view.is_derived_series_feature_enabled)

        flags = {u"soyuz.derived_series_ui.enabled": u"true"}

        with FeatureFixture(flags):

            self.assertTrue(view.is_derived_series_feature_enabled)

    def test_form_hidden_when_derived_series_feature_disabled(self):

        # The form is hidden when the feature flag is not set.

        distroseries = self.factory.makeDistroSeries()

        view = create_initialized_view(distroseries, "+initseries")

        with FeatureFixture({}):

            root = html.fromstring(view())

            self.assertEqual(

                [], root.cssselect("#initseries-form-container"))

            # Instead an explanatory message is shown.

            [message] = root.cssselect("p.error.message")

            self.assertIn(

                u"The Derivative Distributions feature is under development",

                message.text)

    def test_form_shown_when_derived_series_feature_enabled(self):

        # The form is shown when the feature flag is set.

        distroseries = self.factory.makeDistroSeries()

        view = create_initialized_view(distroseries, "+initseries")

        flags = {u"soyuz.derived_series_ui.enabled": u"true"}

        with FeatureFixture(flags):

            root = html.fromstring(view())

            self.assertNotEqual(

                [], root.cssselect("#initseries-form-container"))

            # A different explanatory message is shown for clients that don't

            # process Javascript.

            [message] = root.cssselect("p.error.message")

            self.assertIn(

                u"Javascript is required to use this page",

                message.text)

            self.assertIn(

                u"javascript-disabled",

                message.get("class").split())

    def test_seriesToVocab(self):

        distroseries = self.factory.makeDistroSeries()

        formatted_dict = seriesToVocab(distroseries)

        self.assertEquals(

            ['api_uri', 'title', 'value'],

            sorted(formatted_dict.keys()))