~launchpad-pqm/launchpad/devel

# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the

# GNU Affero General Public License version 3 (see the file LICENSE).

# pylint: disable-msg=E0611,W0212

"""Launchpad bug-related database table classes."""

__metaclass__ = type

__all__ = [

    'Bug',

    'BugAffectsPerson',

    'BugBecameQuestionEvent',

    'BugMute',

    'BugSet',

    'BugTag',

    'FileBugData',

    'get_also_notified_subscribers',

    'get_bug_tags',

    'get_bug_tags_open_count',

    ]

from cStringIO import StringIO

from datetime import (

    datetime,

    timedelta,

    )

from email.Utils import make_msgid

from functools import wraps

from itertools import chain

import operator

import re

from lazr.lifecycle.event import (

    ObjectCreatedEvent,

    ObjectDeletedEvent,

    ObjectModifiedEvent,

    )

from lazr.lifecycle.snapshot import Snapshot

import pytz

from pytz import timezone

from sqlobject import (

    BoolCol,

    ForeignKey,

    IntCol,

    SQLMultipleJoin,

    SQLObjectNotFound,

    SQLRelatedJoin,

    StringCol,

    )

from storm.expr import (

    And,

    Desc,

    In,

    Join,

    LeftJoin,

    Max,

    Not,

    Or,

    Select,

    SQL,

    Sum,

    Union,

    )

from storm.info import ClassAlias

from storm.locals import (

    DateTime,

    Int,

    Reference,

    )

from storm.store import (

    EmptyResultSet,

    Store,

    )

from zope.component import getUtility

from zope.contenttype import guess_content_type

from zope.event import notify

from zope.interface import (

    implements,

    providedBy,

    )

from zope.security.interfaces import Unauthorized

from zope.security.proxy import (

    ProxyFactory,

    removeSecurityProxy,

    )

from lp.answers.interfaces.questiontarget import IQuestionTarget

from lp.app.enums import ServiceUsage

from lp.app.errors import (

    NotFoundError,

    UserCannotUnsubscribePerson,

    )

from lp.app.interfaces.launchpad import ILaunchpadCelebrities

from lp.app.validators import LaunchpadValidationError

from lp.bugs.adapters.bugchange import (

    BranchLinkedToBug,

    BranchUnlinkedFromBug,

    BugConvertedToQuestion,

    BugDuplicateChange,

    BugWatchAdded,

    BugWatchRemoved,

    SeriesNominated,

    UnsubscribedFromBug,

    )

from lp.bugs.enum import BugNotificationLevel

from lp.bugs.errors import (

    BugCannotBePrivate,

    InvalidDuplicateValue,

    SubscriptionPrivacyViolation,

    )

from lp.bugs.interfaces.bug import (

    IBug,

    IBugBecameQuestionEvent,

    IBugMute,

    IBugSet,

    IFileBugData,

    )

from lp.bugs.interfaces.bugactivity import IBugActivitySet

from lp.bugs.interfaces.bugattachment import (

    BugAttachmentType,

    IBugAttachmentSet,

    )

from lp.bugs.interfaces.bugmessage import IBugMessageSet

from lp.bugs.interfaces.bugnomination import (

    BugNominationStatus,

    NominationError,

    NominationSeriesObsoleteError,

    )

from lp.bugs.interfaces.bugnotification import IBugNotificationSet

from lp.bugs.interfaces.bugtask import (

    BugTaskStatus,

    BugTaskStatusSearch,

    IBugTask,

    IBugTaskSet,

    UNRESOLVED_BUGTASK_STATUSES,

    )

from lp.bugs.interfaces.bugtracker import BugTrackerType

from lp.bugs.interfaces.bugwatch import IBugWatchSet

from lp.bugs.interfaces.cve import ICveSet

from lp.bugs.interfaces.hasbug import IHasBug

from lp.bugs.mail.bugnotificationrecipients import BugNotificationRecipients

from lp.bugs.model.bugactivity import BugActivity

from lp.bugs.model.bugattachment import BugAttachment

from lp.bugs.model.bugbranch import BugBranch

from lp.bugs.model.bugcve import BugCve

from lp.bugs.model.bugmessage import BugMessage

from lp.bugs.model.bugnomination import BugNomination

from lp.bugs.model.bugnotification import BugNotification

from lp.bugs.model.bugsubscription import BugSubscription

from lp.bugs.model.bugtarget import OfficialBugTag

from lp.bugs.model.bugtask import (

    BugTask,

    bugtask_sort_key,

    get_bug_privacy_filter,

    )

from lp.bugs.model.bugwatch import BugWatch

from lp.bugs.model.structuralsubscription import (

    get_structural_subscribers,

    get_structural_subscriptions_for_bug,

    )

from lp.code.interfaces.branchcollection import IAllBranches

from lp.hardwaredb.interfaces.hwdb import IHWSubmissionBugSet

from lp.registry.interfaces.accesspolicy import (

    IAccessPolicySource,

    UnsuitableAccessPolicyError,

    )

from lp.registry.interfaces.distribution import IDistribution

from lp.registry.interfaces.distroseries import IDistroSeries

from lp.registry.interfaces.person import (

    IPersonSet,

    validate_person,

    validate_public_person,

    )

from lp.registry.interfaces.product import IProduct

from lp.registry.interfaces.productseries import IProductSeries

from lp.registry.interfaces.role import IPersonRoles

from lp.registry.interfaces.series import SeriesStatus

from lp.registry.interfaces.sourcepackage import ISourcePackage

from lp.registry.model.person import (

    Person,

    person_sort_key,

    PersonSet,

    )

from lp.registry.model.pillar import pillar_sort_key

from lp.registry.model.teammembership import TeamParticipation

from lp.services.config import config

from lp.services.database.constants import UTC_NOW

from lp.services.database.datetimecol import UtcDateTimeCol

from lp.services.database.decoratedresultset import DecoratedResultSet

from lp.services.database.lpstorm import IStore

from lp.services.database.sqlbase import (

    cursor,

    SQLBase,

    sqlvalues,

    )

from lp.services.database.stormbase import StormBase

from lp.services.features import getFeatureFlag

from lp.services.fields import DuplicateBug

from lp.services.helpers import shortlist

from lp.services.librarian.interfaces import ILibraryFileAliasSet

from lp.services.librarian.model import (

    LibraryFileAlias,

    LibraryFileContent,

    )

from lp.services.messages.interfaces.message import (

    IMessage,

    IndexedMessage,

    )

from lp.services.messages.model.message import (

    Message,

    MessageChunk,

    MessageSet,

    )

from lp.services.propertycache import (

    cachedproperty,

    clear_property_cache,

    get_property_cache,

    )

from lp.services.webapp.authorization import check_permission

from lp.services.webapp.interfaces import (

    DEFAULT_FLAVOR,

    ILaunchBag,

    IStoreSelector,

    MAIN_STORE,

    )

_bug_tag_query_template = """

        SELECT %(columns)s FROM %(tables)s WHERE

            %(condition)s GROUP BY BugTag.tag ORDER BY BugTag.tag"""

def snapshot_bug_params(bug_params):

    """Return a snapshot of a `CreateBugParams` object."""

    return Snapshot(

        bug_params, names=[

            "owner", "title", "comment", "description", "msg",

            "datecreated", "security_related", "private",

            "distribution", "sourcepackagename",

            "product", "status", "subscribers", "tags",

            "subscribe_owner", "filed_by", "importance",

            "milestone", "assignee", "cve"])

class BugTag(SQLBase):

    """A tag belonging to a bug."""

    bug = ForeignKey(dbName='bug', foreignKey='Bug', notNull=True)

    tag = StringCol(notNull=True)

def get_bug_tags(context_clause):

    """Return all the bug tags as a list of strings.

    context_clause is a SQL condition clause, limiting the tags to a

    specific context. The SQL clause can only use the BugTask table to

    choose the context.

    """

    from_tables = ['BugTag', 'BugTask']

    select_columns = ['BugTag.tag']

    conditions = ['BugTag.bug = BugTask.bug', '(%s)' % context_clause]

    cur = cursor()

    cur.execute(_bug_tag_query_template % dict(

            columns=', '.join(select_columns),

            tables=', '.join(from_tables),

            condition=' AND '.join(conditions)))

    return shortlist([row[0] for row in cur.fetchall()])

def get_bug_tags_open_count(context_condition, user, tag_limit=0,

    include_tags=None):

    """Worker for IBugTarget.getUsedBugTagsWithOpenCounts.

    See `IBugTarget` for details.

    The only change is that this function takes a SQL expression for limiting

    the found tags.

    :param context_condition: A Storm SQL expression, limiting the

        used tags to a specific context. Only the BugSummary table may be

        used to choose the context. If False then no query will be performed

        (and {} returned).

    """

    # Circular fail.

    from lp.bugs.model.bugsummary import BugSummary

    tags = {}

    if include_tags:

        tags = dict((tag, 0) for tag in include_tags)

    store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)

    admin_team = getUtility(ILaunchpadCelebrities).admin

    if user is not None and not user.inTeam(admin_team):

        store = store.with_(SQL(

            "teams AS ("

            "SELECT team from TeamParticipation WHERE person=?)", (user.id,)))

    where_conditions = [

        BugSummary.status.is_in(UNRESOLVED_BUGTASK_STATUSES),

        BugSummary.tag != None,

        context_condition,

        ]

    if user is None:

        where_conditions.append(BugSummary.viewed_by_id == None)

    elif not user.inTeam(admin_team):

        where_conditions.append(

            Or(

                BugSummary.viewed_by_id == None,

                BugSummary.viewed_by_id.is_in(SQL("SELECT team FROM teams"))

                ))

    sum_count = Sum(BugSummary.count)

    tag_count_columns = (BugSummary.tag, sum_count)

    # Always query for used

    def _query(*args):

        return store.find(tag_count_columns, *(where_conditions + list(args))

            ).group_by(BugSummary.tag).having(sum_count != 0).order_by(

            Desc(Sum(BugSummary.count)), BugSummary.tag)

    used = _query()

    if tag_limit:

        used = used[:tag_limit]

    if include_tags:

        # Union in a query for just include_tags.

        used = used.union(_query(BugSummary.tag.is_in(include_tags)))

    tags.update(dict(used))

    return tags

class BugBecameQuestionEvent:

    """See `IBugBecameQuestionEvent`."""

    implements(IBugBecameQuestionEvent)

    def __init__(self, bug, question, user):

        self.bug = bug

        self.question = question

        self.user = user

class Bug(SQLBase):

    """A bug."""

    implements(IBug)

    _defaultOrder = '-id'

    # db field names

    name = StringCol(unique=True, default=None)

    title = StringCol(notNull=True)

    description = StringCol(notNull=False,

                            default=None)

    owner = ForeignKey(

        dbName='owner', foreignKey='Person',

        storm_validator=validate_public_person, notNull=True)

    duplicateof = ForeignKey(

        dbName='duplicateof', foreignKey='Bug', default=None)

    datecreated = UtcDateTimeCol(notNull=True, default=UTC_NOW)

    date_last_updated = UtcDateTimeCol(notNull=True, default=UTC_NOW)

    private = BoolCol(notNull=True, default=False)

    date_made_private = UtcDateTimeCol(notNull=False, default=None)

    who_made_private = ForeignKey(

        dbName='who_made_private', foreignKey='Person',

        storm_validator=validate_public_person, default=None)

    security_related = BoolCol(notNull=True, default=False)

    access_policy_id = Int(name="access_policy")

    access_policy = Reference(access_policy_id, 'AccessPolicy.id')

    # useful Joins

    activity = SQLMultipleJoin('BugActivity', joinColumn='bug', orderBy='id')

    messages = SQLRelatedJoin('Message', joinColumn='bug',

                           otherColumn='message',

                           intermediateTable='BugMessage',

                           prejoins=['owner'],

                           orderBy=['datecreated', 'id'])

    bug_messages = SQLMultipleJoin(

        'BugMessage', joinColumn='bug', orderBy='index')

    watches = SQLMultipleJoin(

        'BugWatch', joinColumn='bug', orderBy=['bugtracker', 'remotebug'])

    cves = SQLRelatedJoin('Cve', intermediateTable='BugCve',

        orderBy='sequence', joinColumn='bug', otherColumn='cve')

    cve_links = SQLMultipleJoin('BugCve', joinColumn='bug', orderBy='id')

    duplicates = SQLMultipleJoin(

        'Bug', joinColumn='duplicateof', orderBy='id')

    specifications = SQLRelatedJoin('Specification', joinColumn='bug',

        otherColumn='specification', intermediateTable='SpecificationBug',

        orderBy='-datecreated')

    questions = SQLRelatedJoin('Question', joinColumn='bug',

        otherColumn='question', intermediateTable='QuestionBug',

        orderBy='-datecreated')

    linked_branches = SQLMultipleJoin(

        'BugBranch', joinColumn='bug', orderBy='id')

    date_last_message = UtcDateTimeCol(default=None)

    number_of_duplicates = IntCol(notNull=True, default=0)

    message_count = IntCol(notNull=True, default=0)

    users_affected_count = IntCol(notNull=True, default=0)

    users_unaffected_count = IntCol(notNull=True, default=0)

    heat = IntCol(notNull=True, default=0)

    heat_last_updated = UtcDateTimeCol(default=None)

    latest_patch_uploaded = UtcDateTimeCol(default=None)

    @cachedproperty

    def _subscriber_cache(self):

        """Caches known subscribers."""

        return set()

    @cachedproperty

    def _subscriber_dups_cache(self):

        """Caches known subscribers to dupes."""

        return set()

    @cachedproperty

    def _unsubscribed_cache(self):

        """Cache known non-subscribers."""

        return set()

    @property

    def latest_patch(self):

        """See `IBug`."""

        # We want to retrieve the most recently added bug attachment

        # that is of type BugAttachmentType.PATCH. In order to find

        # this attachment, we should in theory sort by

        # BugAttachment.message.datecreated. Since we don't have

        # an index for Message.datecreated, such a query would be

        # quite slow. We search instead for the BugAttachment with

        # the largest ID for a given bug. This is "nearly" equivalent

        # to searching the record with the maximum value of

        # message.datecreated: The only exception is the rare case when

        # two BugAttachment records are simultaneuosly added to the same

        # bug, where bug_attachment_1.id < bug_attachment_2.id, while

        # the Message record for bug_attachment_2 is created before

        # the Message record for bug_attachment_1. The difference of

        # the datecreated values of the Message records is in this case

        # probably smaller than one second and the selection of the

        # "most recent" patch anyway somewhat arbitrary.

        return Store.of(self).find(

            BugAttachment, BugAttachment.id == Select(

                Max(BugAttachment.id),

                And(BugAttachment.bug == self.id,

                    BugAttachment.type == BugAttachmentType.PATCH))).one()

    @property

    def comment_count(self):

        """See `IBug`."""

        return self.message_count - 1

    @property

    def users_affected(self):

        """See `IBug`."""

        return Store.of(self).find(

            Person, BugAffectsPerson.person == Person.id,

            BugAffectsPerson.affected,

            BugAffectsPerson.bug == self)

    @property

    def users_unaffected(self):

        """See `IBug`."""

        return Store.of(self).find(

            Person, BugAffectsPerson.person == Person.id,

            Not(BugAffectsPerson.affected),

            BugAffectsPerson.bug == self)

    @property

    def user_ids_affected_with_dupes(self):

        """Return all IDs of Persons affected by this bug and its dupes.

        The return value is a Storm expression.  Running a query with

        this expression returns a result that may contain the same ID

        multiple times, for example if that person is affected via

        more than one duplicate."""

        return Union(

            Select(Person.id,

                   And(BugAffectsPerson.person == Person.id,

                       BugAffectsPerson.affected,

                       BugAffectsPerson.bug == self)),

            Select(Person.id,

                   And(BugAffectsPerson.person == Person.id,

                       BugAffectsPerson.bug == Bug.id,

                       BugAffectsPerson.affected,

                       Bug.duplicateof == self.id)))

    @property

    def users_affected_with_dupes(self):

        """See `IBug`."""

        return Store.of(self).find(

            Person,

            Person.id.is_in(self.user_ids_affected_with_dupes))

    @property

    def users_affected_count_with_dupes(self):

        """See `IBug`."""

        return self.users_affected_with_dupes.count()

    @property

    def other_users_affected_count_with_dupes(self):

        """See `IBug`."""

        current_user = getUtility(ILaunchBag).user

        if not current_user:

            return self.users_affected_count_with_dupes

        return self.users_affected_with_dupes.find(

            Person.id != current_user.id).count()

    @property

    def indexed_messages(self):

        """See `IMessageTarget`."""

        # Note that this is a decorated result set, so will cache its

        # value (in the absence of slices)

        return self._indexed_messages(include_content=True)

    def _indexed_messages(self, include_content=False, include_parents=True):

        """Get the bugs messages, indexed.

        :param include_content: If True retrieve the content for the messages

            too.

        :param include_parents: If True retrieve the object for parent

            messages too. If False the parent attribute will be *forced* to

            None to reduce database lookups.

        """

        # Make all messages be 'in' the main bugtask.

        inside = self.default_bugtask

        store = Store.of(self)

        message_by_id = {}

        to_messages = lambda rows: [row[0] for row in rows]

        def eager_load_owners(messages):

            # Because we may have multiple owners, we spend less time

            # in storm with very large bugs by not joining and instead

            # querying a second time. If this starts to show high db

            # time, we can left outer join instead.

            owner_ids = set(message.ownerID for message in messages)

            owner_ids.discard(None)

            if not owner_ids:

                return

            list(store.find(Person, Person.id.is_in(owner_ids)))

        def eager_load_content(messages):

            # To avoid the complexity of having multiple rows per

            # message, or joining in the database (though perhaps in

            # future we should do that), we do a single separate query

            # for the message content.

            message_ids = set(message.id for message in messages)

            chunks = store.find(

                MessageChunk, MessageChunk.messageID.is_in(message_ids))

            chunks.order_by(MessageChunk.id)

            chunk_map = {}

            for chunk in chunks:

                message_chunks = chunk_map.setdefault(chunk.messageID, [])

                message_chunks.append(chunk)

            for message in messages:

                if message.id not in chunk_map:

                    continue

                cache = get_property_cache(message)

                cache.text_contents = Message.chunks_text(

                    chunk_map[message.id])

        def eager_load(rows):

            messages = to_messages(rows)

            eager_load_owners(messages)

            if include_content:

                eager_load_content(messages)

        def index_message(row):

            # convert row to an IndexedMessage

            if include_parents:

                message, parent, bugmessage = row

                if parent is not None:

                    # If there is an IndexedMessage available as parent, use

                    # that to reduce on-demand parent lookups.

                    parent = message_by_id.get(parent.id, parent)

            else:

                message, bugmessage = row

                parent = None  # parent attribute is not going to be accessed.

            index = bugmessage.index

            result = IndexedMessage(message, inside, index, parent)

            if include_parents:

                # This message may be the parent for another: stash it to

                # permit use.

                message_by_id[message.id] = result

            return result

        if include_parents:

            ParentMessage = ClassAlias(Message)

            ParentBugMessage = ClassAlias(BugMessage)

            tables = [

                Message,

                Join(

                    BugMessage,

                    BugMessage.messageID == Message.id),

                LeftJoin(

                    Join(

                        ParentMessage,

                        ParentBugMessage,

                        ParentMessage.id == ParentBugMessage.messageID),

                    And(

                        Message.parent == ParentMessage.id,

                        ParentBugMessage.bugID == self.id)),

                ]

            results = store.using(*tables).find(

                (Message, ParentMessage, BugMessage),

                BugMessage.bugID == self.id,

                )

        else:

            lookup = Message, BugMessage

            results = store.find(lookup,

                BugMessage.bugID == self.id,

                BugMessage.messageID == Message.id,

                )

        results.order_by(BugMessage.index)

        return DecoratedResultSet(results, index_message,

            pre_iter_hook=eager_load)

    @property

    def displayname(self):

        """See `IBug`."""

        dn = 'Bug #%d' % self.id

        if self.name:

            dn += ' (' + self.name + ')'

        return dn

    @cachedproperty

    def bugtasks(self):

        """See `IBug`."""

        # \o/ circular imports.

        from lp.registry.model.distribution import Distribution

        from lp.registry.model.distroseries import DistroSeries

        from lp.registry.model.product import Product

        from lp.registry.model.productseries import ProductSeries

        from lp.registry.model.sourcepackagename import SourcePackageName

        store = Store.of(self)

        tasks = list(store.find(BugTask, BugTask.bugID == self.id))

        # The bugtasks attribute is iterated in the API and web

        # services, so it needs to preload all related data otherwise

        # late evaluation is triggered in both places. Separately,

        # bugtask_sort_key requires the related products, series,

        # distros, distroseries and source package names to be loaded.

        ids = set(map(operator.attrgetter('assigneeID'), tasks))

        ids.update(map(operator.attrgetter('ownerID'), tasks))

        ids.discard(None)

        if ids:

            list(getUtility(IPersonSet).getPrecachedPersonsFromIDs(

                ids, need_validity=True))

        def load_something(attrname, klass):

            ids = set(map(operator.attrgetter(attrname), tasks))

            ids.discard(None)

            if not ids:

                return

            list(store.find(klass, klass.id.is_in(ids)))

        load_something('productID', Product)

        load_something('productseriesID', ProductSeries)

        load_something('distributionID', Distribution)

        load_something('distroseriesID', DistroSeries)

        load_something('sourcepackagenameID', SourcePackageName)

        list(store.find(BugWatch, BugWatch.bugID == self.id))

        return sorted(tasks, key=bugtask_sort_key)