~launchpad-pqm/launchpad/devel

# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the

# GNU Affero General Public License version 3 (see the file LICENSE).

# pylint: disable-msg=W0231,E1002

"""Definition of the internet servers that Launchpad uses."""

__metaclass__ = type

import cgi

import threading

import xmlrpclib

from lazr.restful.interfaces import (

    ICollectionResource,

    IWebServiceConfiguration,

    IWebServiceVersion,

    )

from lazr.restful.publisher import (

    WebServicePublicationMixin,

    WebServiceRequestTraversal,

    )

from lazr.uri import URI

import transaction

from transaction.interfaces import ISynchronizer

from zc.zservertracelog.tracelog import Server as ZServerTracelogServer

from zope.app.form.browser.itemswidgets import MultiDataHelper

from zope.app.form.browser.widget import SimpleInputWidget

from zope.app.publication.httpfactory import HTTPPublicationRequestFactory

from zope.app.publication.interfaces import IRequestPublicationFactory

from zope.app.publication.requestpublicationregistry import (

    factoryRegistry as publisher_factory_registry,

    )

from zope.app.server import wsgi

from zope.app.wsgi import WSGIPublisherApplication

from zope.component import getUtility

from zope.event import notify

from zope.interface import (

    alsoProvides,

    implements,

    )

from zope.publisher.browser import (

    BrowserRequest,

    BrowserResponse,

    TestRequest,

    )

from zope.publisher.interfaces import NotFound

from zope.publisher.xmlrpc import (

    XMLRPCRequest,

    XMLRPCResponse,

    )

from zope.security.interfaces import (

    IParticipation,

    Unauthorized,

    )

from zope.security.proxy import (

    isinstance as zope_isinstance,

    removeSecurityProxy,

    )

from zope.server.http.commonaccesslogger import CommonAccessLogger

from zope.server.http.wsgihttpserver import PMDBWSGIHTTPServer

from zope.session.interfaces import ISession

from lp.app.errors import UnexpectedFormData

import lp.layers

from lp.services.config import config

from lp.services.features import get_relevant_feature_controller

from lp.services.features.flags import NullFeatureController

from lp.services.feeds.interfaces.application import IFeedsApplication

from lp.services.feeds.interfaces.feed import IFeed

from lp.services.oauth.interfaces import (

    IOAuthConsumerSet,

    IOAuthSignedRequest,

    TokenException,

    )

from lp.services.propertycache import cachedproperty

from lp.services.webapp.authentication import (

    check_oauth_signature,

    get_oauth_authorization,

    )

from lp.services.webapp.authorization import (

    LAUNCHPAD_SECURITY_POLICY_CACHE_KEY,

    )

from lp.services.webapp.errorlog import ErrorReportRequest

from lp.services.webapp.interfaces import (

    FinishReadOnlyRequestEvent,

    IAPIDocRoot,

    IBasicLaunchpadRequest,

    IBrowserFormNG,

    ILaunchpadBrowserApplicationRequest,

    ILaunchpadProtocolError,

    INotificationRequest,

    INotificationResponse,

    IPlacelessAuthUtility,

    IPlacelessLoginSource,

    OAuthPermission,

    )

from lp.services.webapp.notifications import (

    NotificationList,

    NotificationRequest,

    NotificationResponse,

    )

from lp.services.webapp.opstats import OpStats

from lp.services.webapp.publication import LaunchpadBrowserPublication

from lp.services.webapp.publisher import (

    get_current_browser_request,

    RedirectionView,

    )

from lp.services.webapp.vhosts import allvhosts

from lp.services.webservice.interfaces import IWebServiceApplication

from lp.testopenid.interfaces.server import ITestOpenIDApplication

from lp.xmlrpc.interfaces import IPrivateApplication

class StepsToGo:

    """

    >>> class FakeRequest:

    ...     def __init__(self, traversed, stack):

    ...         self._traversed_names = traversed

    ...         self.stack = stack

    ...     def getTraversalStack(self):

    ...         return self.stack

    ...     def setTraversalStack(self, stack):

    ...         self.stack = stack

    >>> request = FakeRequest([], ['baz', 'bar', 'foo'])

    >>> stepstogo = StepsToGo(request)

    >>> stepstogo.startswith()

    True

    >>> stepstogo.startswith('foo')

    True

    >>> stepstogo.startswith('foo', 'bar')

    True

    >>> stepstogo.startswith('foo', 'baz')

    False

    >>> len(stepstogo)

    3

    >>> print stepstogo.consume()

    foo

    >>> request._traversed_names

    ['foo']

    >>> request.stack

    ['baz', 'bar']

    >>> print stepstogo.consume()

    bar

    >>> bool(stepstogo)

    True

    >>> print stepstogo.consume()

    baz

    >>> print stepstogo.consume()

    None

    >>> bool(stepstogo)

    False

    >>> request = FakeRequest([], ['baz', 'bar', 'foo'])

    >>> list(StepsToGo(request))

    ['foo', 'bar', 'baz']

    """

    @property

    def _stack(self):

        return self.request.getTraversalStack()

    def __init__(self, request):

        self.request = request

    def __iter__(self):

        return self

    def consume(self):

        """Remove the next path step and return it.

        Returns None if there are no path steps left.

        """

        stack = self.request.getTraversalStack()

        try:

            nextstep = stack.pop()

        except IndexError:

            return None

        self.request._traversed_names.append(nextstep)

        self.request.setTraversalStack(stack)

        return nextstep

    def next(self):

        value = self.consume()

        if value is None:

            raise StopIteration

        return value

    def startswith(self, *args):

        """Return whether the steps to go start with the names given."""

        if not args:

            return True

        return self._stack[-len(args):] == list(reversed(args))

    def __len__(self):

        return len(self._stack)

    def __nonzero__(self):

        return bool(self._stack)

class ApplicationServerSettingRequestFactory:

    """Create a request and call its setApplicationServer method.

    Due to the factory-fanatical design of this part of Zope3, we need

    to have a kind of proxying factory here so that we can create an

    appropriate request and call its setApplicationServer method before it

    is used.

    """

    def __init__(self, requestfactory, host, protocol, port):

        self.requestfactory = requestfactory

        self.host = host

        self.protocol = protocol

        self.port = port

    def __call__(self, body_instream, environ, response=None):

        """Equivalent to the request's __init__ method."""

        # Make sure that HTTPS variable is set so that request.getURL() is

        # sane

        if self.protocol == 'https':

            environ['HTTPS'] = 'on'

        request = self.requestfactory(body_instream, environ, response)

        request.setApplicationServer(self.host, self.protocol, self.port)

        return request

class VirtualHostRequestPublicationFactory:

    """An `IRequestPublicationFactory` handling request to a Launchpad vhost.

    This factory will accepts requests to a particular Launchpad virtual host

    that matches a particular port and set of HTTP methods.

    """

    implements(IRequestPublicationFactory)

    default_methods = ['GET', 'HEAD', 'POST']

    def __init__(self, vhost_name, request_factory, publication_factory,

                 port=None, methods=None, handle_default_host=False):

        """Creates a new factory.

        :param vhost_name: The config section defining the virtual host

             handled by this factory.

        :param request_factory: The request factory to use for this virtual

             host's requests.

        :param publication_factory: The publication factory to use for this

            virtual host's requests.

        :param port: The port which is handled by this factory. If

            this is None, this factory will handle requests that

            originate on any port.

        :param methods: A sequence of HTTP methods that this factory handles.

        :param handle_default_host: Whether or not this factory is

            capable of handling requests that specify no hostname.

        """

        self.vhost_name = vhost_name

        self.request_factory = request_factory

        self.publication_factory = publication_factory

        self.port = port

        if methods is None:

            methods = self.default_methods

        self.methods = methods

        self.handle_default_host = handle_default_host

        self.vhost_config = allvhosts.configs[self.vhost_name]

        self.all_hostnames = set(self.vhost_config.althostnames

                                 + [self.vhost_config.hostname])

        self._thread_local = threading.local()

        self._thread_local.environment = None

    def canHandle(self, environment):

        """See `IRequestPublicationFactory`.

        Returns true if the HTTP host and port of the incoming request

        match the ones this factory is equipped to handle.

        """

        # We look at the wsgi environment to get the port this request

        # is coming in over.  The port number can be in one of two

        # places; either it's on the SERVER_PORT environment variable

        # or, as is the case with the test suite, it's on the

        # HTTP_HOST variable after a colon.

        # The former takes precedence, the port from the host variable is

        # only checked because the test suite doesn't set SERVER_PORT.

        host = environment.get('HTTP_HOST', '')

        port = environment.get('SERVER_PORT')

        if ":" in host:

            assert len(host.split(':')) == 2, (

                "Having a ':' in the host name isn't allowed.")

            host, new_port = host.split(':')

            if port is None:

                port = new_port

        if host == '':

            if not self.handle_default_host:

                return False

        elif host not in self.all_hostnames:

            return False

        else:

            # This factory handles this host.

            pass

        if self.port is not None:

            if port is not None:

                try:

                    port = int(port)

                except (ValueError):

                    port = None

            if self.port != port:

                return False

        self._thread_local.environment = environment

        self._thread_local.host = host

        return True

    def __call__(self):

        """See `IRequestPublicationFactory`.

        We know that this factory is the right one for the given host

        and port. But there might be something else wrong with the

        request.  For instance, it might have the wrong HTTP method.

        """

        environment = self._thread_local.environment

        if environment is None:

            raise AssertionError('This factory declined the request.')

        root_url = URI(self.vhost_config.rooturl)

        real_request_factory, publication_factory = (

            self.checkRequest(environment))

        if not real_request_factory:

            real_request_factory, publication_factory = (

                self.getRequestAndPublicationFactories(environment))

        host = environment.get('HTTP_HOST', '').split(':')[0]

        if host in ['', 'localhost']:

            # Sometimes requests come in to the default or local host.

            # If we set the application server for these requests,

            # they'll be handled as launchpad.net requests, and

            # responses will go out containing launchpad.net URLs.

            # That's a little unelegant, so we don't set the application

            # server for these requests.

            request_factory = real_request_factory

        else:

            request_factory = ApplicationServerSettingRequestFactory(

                real_request_factory,

                root_url.host,

                root_url.scheme,

                root_url.port)

        self._thread_local.environment = None

        return (request_factory, publication_factory)

    def getRequestAndPublicationFactories(self, environment):

        """Return the request and publication factories to use.

        You can override this method if the request and publication can

        vary based on the environment.

        """

        return self.request_factory, self.publication_factory

    def getAcceptableMethods(self, environment):

        """Return the HTTP methods acceptable in this particular environment.

        """

        return self.methods

    def checkRequest(self, environment):

        """Makes sure that the incoming HTTP request is of an expected type.

        This is different from canHandle() because we know the request

        went to the right place. It's just that it might be an invalid

        request for this handler.

        :return: An appropriate ProtocolErrorPublicationFactory if the

            HTTP request doesn't comply with the expected protocol. If

            the request does comply, (None, None).

        """

        method = environment.get('REQUEST_METHOD')

        if method in self.getAcceptableMethods(environment):

            factories = (None, None)

        else:

            request_factory = ProtocolErrorRequest

            publication_factory = ProtocolErrorPublicationFactory(

                405, headers={'Allow': " ".join(self.methods)})

            factories = (request_factory, publication_factory)

        return factories

class XMLRPCRequestPublicationFactory(VirtualHostRequestPublicationFactory):

    """A VirtualHostRequestPublicationFactory for XML-RPC.

    This factory only accepts XML-RPC method calls.

    """

    def __init__(self, vhost_name, request_factory, publication_factory,

                 port=None):

        super(XMLRPCRequestPublicationFactory, self).__init__(

            vhost_name, request_factory, publication_factory, port, ['POST'])

    def checkRequest(self, environment):

        """See `VirtualHostRequestPublicationFactory`.

        Accept only requests where the MIME type is text/xml.

        """

        request_factory, publication_factory = (

            super(XMLRPCRequestPublicationFactory, self).checkRequest(

                environment))

        if request_factory is None:

            mime_type = environment.get('CONTENT_TYPE')

            if mime_type != 'text/xml':

                request_factory = ProtocolErrorRequest

                # 415 - Unsupported Media Type

                publication_factory = ProtocolErrorPublicationFactory(415)

        return request_factory, publication_factory

class WebServiceRequestPublicationFactory(

    VirtualHostRequestPublicationFactory):

    """A VirtualHostRequestPublicationFactory for requests against

    resources published through a web service.

    """

    default_methods = [

        'GET', 'HEAD', 'POST', 'PATCH', 'PUT', 'DELETE', 'OPTIONS']

    def __init__(self, vhost_name, request_factory, publication_factory,

                 port=None):

        """This factory accepts requests that use all five major HTTP methods.

        """

        super(WebServiceRequestPublicationFactory, self).__init__(

            vhost_name, request_factory, publication_factory, port)

class VHostWebServiceRequestPublicationFactory(

    VirtualHostRequestPublicationFactory):

    """An `IRequestPublicationFactory` handling requests to vhosts.

    It also handles requests to the launchpad web service, if the

    request's path points to a web service resource.

    """

    def getAcceptableMethods(self, environment):

        """See `VirtualHostRequestPublicationFactory`.

        If this is a request for a webservice path, returns the appropriate

        methods.

        """

        if self.isWebServicePath(environment.get('PATH_INFO', '')):

            return WebServiceRequestPublicationFactory.default_methods

        else:

            return super(

                VHostWebServiceRequestPublicationFactory,

                self).getAcceptableMethods(environment)

    def getRequestAndPublicationFactories(self, environment):

        """See `VirtualHostRequestPublicationFactory`.

        If this is a request for a webservice path, returns the appropriate

        factories.

        """

        if self.isWebServicePath(environment.get('PATH_INFO', '')):

            return WebServiceClientRequest, WebServicePublication

        else:

            return super(

                VHostWebServiceRequestPublicationFactory,

                self).getRequestAndPublicationFactories(environment)

    def isWebServicePath(self, path):

        """Does the path refer to a web service resource?"""

        # Add a trailing slash, if it is missing.

        if not path.endswith('/'):

            path = path + '/'

        ws_config = getUtility(IWebServiceConfiguration)

        return path.startswith('/%s/' % ws_config.path_override)

class NotFoundRequestPublicationFactory:

    """An IRequestPublicationFactory which always yields a 404."""

    def canHandle(self, environment):

        """See `IRequestPublicationFactory`."""

        return True

    def __call__(self):

        """See `IRequestPublicationFactory`.

        Unlike other publication factories, this one doesn't wrap its

        request factory in an ApplicationServerSettingRequestFactory.

        That's because it's only triggered when there's no valid hostname.

        """

        return (ProtocolErrorRequest, ProtocolErrorPublicationFactory(404))

def get_query_string_params(request):

    """Return a dict of the decoded query string params for a request.

    The parameter values will be decoded as unicodes, exactly as

    `BrowserRequest` would do to build the request form.

    Defined here so that it can be used in both BasicLaunchpadRequest and

    the LaunchpadTestRequest (which doesn't inherit from

    BasicLaunchpadRequest).

    """

    query_string = request.get('QUERY_STRING', '')

    # Just in case QUERY_STRING is in the environment explicitly as

    # None (Some tests seem to do this, but not sure if it can ever

    # happen outside of tests.)

    if query_string is None:

        query_string = ''

    parsed_qs = cgi.parse_qs(query_string, keep_blank_values=True)

    # Use BrowserRequest._decode() for decoding the received parameters.

    decoded_qs = {}

    for key, values in parsed_qs.iteritems():

        decoded_qs[key] = [

            request._decode(value) for value in values]

    return decoded_qs

class LaunchpadBrowserRequestMixin:

    """Provides methods used for both API and web browser requests."""

    def getRootURL(self, rootsite):

        """See IBasicLaunchpadRequest."""

        if rootsite is not None:

            assert rootsite in allvhosts.configs, (

                "rootsite is %s.  Must be in %r." % (

                    rootsite, sorted(allvhosts.configs.keys())))

            root_url = allvhosts.configs[rootsite].rooturl

        else:

            root_url = self.getApplicationURL() + '/'

        return root_url

    @property

    def is_ajax(self):

        """See `IBasicLaunchpadRequest`."""

        return 'XMLHttpRequest' == self.getHeader('HTTP_X_REQUESTED_WITH')

    def getURL(self, level=0, path_only=False, include_query=False):

        """See `IBasicLaunchpadRequest`."""

        sup = super(LaunchpadBrowserRequestMixin, self)

        url = sup.getURL(level, path_only)

        if include_query:

            query_string = self.get('QUERY_STRING')

            if query_string is not None and len(query_string) > 0:

                url = "%s?%s" % (url, query_string)

        return url

class BasicLaunchpadRequest(LaunchpadBrowserRequestMixin):

    """Mixin request class to provide stepstogo."""

    implements(IBasicLaunchpadRequest)

    def __init__(self, body_instream, environ, response=None):

        self.traversed_objects = []

        self._wsgi_keys = set()

        self.needs_datepicker_iframe = False

        self.needs_datetimepicker_iframe = False

        super(BasicLaunchpadRequest, self).__init__(

            body_instream, environ, response)

        # Our response always vary based on authentication.

        self.response.setHeader('Vary', 'Cookie, Authorization')

    @property

    def stepstogo(self):

        return StepsToGo(self)

    def retry(self):

        """See IPublisherRequest."""

        new_request = super(BasicLaunchpadRequest, self).retry()

        # Propagate the list of keys we have set in the WSGI environment.

        new_request._wsgi_keys = self._wsgi_keys

        return new_request

    def getNearest(self, *some_interfaces):

        """See ILaunchpadBrowserApplicationRequest.getNearest()"""

        for context in reversed(self.traversed_objects):

            for iface in some_interfaces:

                if iface.providedBy(context):

                    return context, iface

        return None, None

    def setInWSGIEnvironment(self, key, value):

        """Set a key-value pair in the WSGI environment of this request.

        Raises KeyError if the key is already present in the environment

        but not set with setInWSGIEnvironment().

        """

        # This method expects the BasicLaunchpadRequest mixin to be used

        # with a base that provides self._orig_env.

        if key not in self._wsgi_keys and key in self._orig_env:

            raise KeyError("'%s' already present in wsgi environment." % key)

        self._orig_env[key] = value

        self._wsgi_keys.add(key)

    @cachedproperty

    def query_string_params(self):

        """See ILaunchpadBrowserApplicationRequest."""

        return get_query_string_params(self)

class LaunchpadBrowserRequest(BasicLaunchpadRequest, BrowserRequest,

                              NotificationRequest, ErrorReportRequest):

    """Integration of launchpad mixin request classes to make an uber

    launchpad request class.

    """

    implements(

        ILaunchpadBrowserApplicationRequest, ISynchronizer,

        lp.layers.LaunchpadLayer)

    retry_max_count = 5    # How many times we're willing to retry

    def __init__(self, body_instream, environ, response=None):

        BasicLaunchpadRequest.__init__(self, body_instream, environ, response)

        transaction.manager.registerSynch(self)

    def _createResponse(self):

        """As per zope.publisher.browser.BrowserRequest._createResponse"""

        return LaunchpadBrowserResponse()

    @cachedproperty

    def form_ng(self):

        """See ILaunchpadBrowserApplicationRequest."""

        return BrowserFormNG(self.form)

    def setPrincipal(self, principal):

        self.clearSecurityPolicyCache()

        BrowserRequest.setPrincipal(self, principal)

    def clearSecurityPolicyCache(self):

        if LAUNCHPAD_SECURITY_POLICY_CACHE_KEY in self.annotations:

            del self.annotations[LAUNCHPAD_SECURITY_POLICY_CACHE_KEY]

    def beforeCompletion(self, transaction):

        """See `ISynchronizer`."""

        pass

    def afterCompletion(self, transaction):

        """See `ISynchronizer`.

        We clear the cache of security policy results on commit, as objects

        will be refetched from the database and the security checks may result

        in different answers.

        """

        self.clearSecurityPolicyCache()

    def newTransaction(self, transaction):

        """See `ISynchronizer`."""

        pass

class BrowserFormNG:

    """Wrapper that provides IBrowserFormNG around a regular form dict."""

    implements(IBrowserFormNG)

    def __init__(self, form):

        """Create a new BrowserFormNG that wraps a dict containing form data.

        """

        self.form = form

    def __contains__(self, name):

        """See IBrowserFormNG."""

        return name in self.form

    def __iter__(self):

        """See IBrowserFormNG."""

        return iter(self.form)

    def getOne(self, name, default=None):

        """See IBrowserFormNG."""

        value = self.form.get(name, default)

        if zope_isinstance(value, (list, tuple)):

            raise UnexpectedFormData(

                'Expected only one value form field %s: %s' % (name, value))

        return value

    def getAll(self, name, default=None):

        """See IBrowserFormNG."""

        # We don't want a mutable as a default parameter, so we use None as a

        # marker.

        if default is None:

            default = []

        else:

            assert zope_isinstance(default, list), (

                "default should be a list: %s" % default)

        value = self.form.get(name, default)

        if not zope_isinstance(value, list):

            value = [value]

        return value

def web_service_request_to_browser_request(webservice_request):

    """Convert a given webservice request into a webapp one.

    Overrides 'SERVER_URL' to the 'mainsite', preserving headers and

    body.  Encodes PATH_INFO because it is unconditionally decoded by

    zope.publisher.http.sane_environment.

    """

    body = webservice_request.bodyStream.getCacheStream().read()

    environ = dict(webservice_request.environment)

    environ['SERVER_URL'] = allvhosts.configs['mainsite'].rooturl

    if 'PATH_INFO' in environ:

        environ['PATH_INFO'] = environ['PATH_INFO'].encode('utf-8')

    return LaunchpadBrowserRequest(body, environ)

class Zope3WidgetsUseIBrowserFormNGMonkeyPatch:

    """Make Zope3 widgets use IBrowserFormNG.

    Replace the SimpleInputWidget._getFormInput method with one using

    `IBrowserFormNG`.

    """

    installed = False

    @classmethod

    def install(cls):

        """Install the monkey patch."""

        assert not cls.installed, "Monkey patch is already installed."