← Back to branch summary

~launchpad-pqm/launchpad/devel

« back to all changes in this revision

Viewing changes to lib/canonical/testing/layers.py

  • Committer: Julian Edwards
  • Date: 2011-07-28 20:46:18 UTC
  • mfrom: (13553 devel)
  • mto: This revision was merged to the branch mainline in revision 13555.
  • Revision ID: julian.edwards@canonical.com-20110728204618-tivj2wx2oa9s32bx
merge trunk

Show diffs side-by-side

added added

removed removed

Lines of Context:
lib/canonical/testing/layers.py
1
 
# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
 
1
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2
2
# GNU Affero General Public License version 3 (see the file LICENSE).
3
3
 
4
4
# We like global!
44
44
    'TwistedLaunchpadZopelessLayer',
45
45
    'TwistedLayer',
46
46
    'YUITestLayer',
47
 
    'YUIAppServerLayer',
48
47
    'ZopelessAppServerLayer',
49
48
    'ZopelessDatabaseLayer',
50
49
    'ZopelessLayer',
51
50
    'disconnect_stores',
52
51
    'reconnect_stores',
53
 
    'wsgi_application',
54
52
    ]
55
53
 
56
54
from cProfile import Profile
63
61
import socket
64
62
import subprocess
65
63
import sys
66
 
import tempfile
67
64
from textwrap import dedent
68
65
import threading
69
66
import time
95
92
    )
96
93
from zope.component.interfaces import ComponentLookupError
97
94
import zope.publisher.publish
98
 
from zope.security.management import (
99
 
    endInteraction,
100
 
    getSecurityPolicy,
101
 
    )
 
95
from zope.security.management import getSecurityPolicy
 
96
from zope.security.simplepolicies import PermissiveSecurityPolicy
102
97
from zope.server.logger.pythonlogger import PythonLogger
103
98
 
104
99
from canonical.config import (
110
105
    ConfigFixture,
111
106
    ConfigUseFixture,
112
107
    )
113
 
from canonical.database.sqlbase import session_store
114
 
from lp.services.scripts import execute_zcml_for_scripts
115
 
from canonical.launchpad.webapp.authorization import (
116
 
    LaunchpadPermissiveSecurityPolicy,
117
 
    )
 
108
from canonical.database.revision import (
 
109
    confirm_dbrevision,
 
110
    confirm_dbrevision_on_startup,
 
111
    )
 
112
from canonical.database.sqlbase import (
 
113
    cursor,
 
114
    session_store,
 
115
    ZopelessTransactionManager,
 
116
    )
 
117
from canonical.launchpad.interfaces.mailbox import IMailBox
 
118
from canonical.launchpad.scripts import execute_zcml_for_scripts
118
119
from canonical.launchpad.webapp.interfaces import (
119
120
    DEFAULT_FLAVOR,
120
121
    IOpenLaunchBag,
133
134
    set_default_timeout_function,
134
135
    )
135
136
from canonical.librarian.testing.server import LibrarianServerFixture
 
137
from canonical.lp import initZopeless
136
138
from canonical.testing import reset_logging
137
139
from canonical.testing.profiled import profiled
138
140
from canonical.testing.smtpd import SMTPController
139
141
from lp.services.googlesearch.tests.googleserviceharness import (
140
142
    GoogleServiceTestSetup,
141
143
    )
142
 
from lp.services.mail.mailbox import (
143
 
    IMailBox,
144
 
    TestMailBox,
145
 
    )
146
 
from lp.services.mail.sendmail import set_immediate_mail_delivery
 
144
from lp.services.mail.mailbox import TestMailBox
147
145
import lp.services.mail.stub
148
146
from lp.services.memcache.client import memcache_client_factory
149
147
from lp.services.osutils import kill_by_pidfile
150
 
from lp.services.rabbit.server import RabbitServer
151
148
from lp.testing import (
152
149
    ANONYMOUS,
 
150
    is_logged_in,
153
151
    login,
154
152
    logout,
155
153
    )
156
 
from lp.testing.dbuser import switch_dbuser
 
154
from lp.testing.fixture import RabbitServer
157
155
from lp.testing.pgsql import PgTestSetup
158
156
 
159
157
 
215
213
            store.close()
216
214
 
217
215
 
218
 
def reconnect_stores(reset=False):
 
216
def reconnect_stores(database_config_section='launchpad'):
219
217
    """Reconnect Storm stores, resetting the dbconfig to its defaults.
220
218
 
221
219
    After reconnecting, the database revision will be checked to make
222
220
    sure the right data is available.
223
221
    """
224
222
    disconnect_stores()
225
 
    if reset:
226
 
        dbconfig.reset()
 
223
    dbconfig.setConfigSection(database_config_section)
227
224
 
228
225
    main_store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
229
226
    assert main_store is not None, 'Failed to reconnect'
230
227
 
 
228
    # Confirm the database has the right patchlevel
 
229
    confirm_dbrevision(cursor())
 
230
 
231
231
    # Confirm that SQLOS is again talking to the database (it connects
232
232
    # as soon as SQLBase._connection is accessed
233
233
    r = main_store.execute('SELECT count(*) FROM LaunchpadDatabaseRevision')
487
487
                "Component architecture should not be loaded by tests. "
488
488
                "This should only be loaded by the Layer.")
489
489
 
 
490
        # Detect a test that installed the Zopeless database adapter
 
491
        # but failed to unregister it. This could be done automatically,
 
492
        # but it is better for the tear down to be explicit.
 
493
        if ZopelessTransactionManager._installed is not None:
 
494
            raise LayerIsolationError(
 
495
                "Zopeless environment was setup and not torn down.")
 
496
 
490
497
        # Detect a test that forgot to reset the default socket timeout.
491
498
        # This safety belt is cheap and protects us from very nasty
492
499
        # intermittent test failures: see bug #140068 for an example.
599
606
            ]
600
607
        if config.memcached.verbose:
601
608
            cmd.append('-vv')
602
 
            stdout = sys.stdout
603
 
            stderr = sys.stderr
604
 
        else:
605
 
            stdout = tempfile.NamedTemporaryFile()
606
 
            stderr = tempfile.NamedTemporaryFile()
607
609
        MemcachedLayer._memcached_process = subprocess.Popen(
608
 
            cmd, stdin=subprocess.PIPE, stdout=stdout, stderr=stderr)
 
610
            cmd, stdin=subprocess.PIPE)
609
611
        MemcachedLayer._memcached_process.stdin.close()
610
612
 
611
613
        # Wait for the memcached to become operational.
754
756
        cls.force_dirty_database()
755
757
        cls._db_fixture.tearDown()
756
758
        cls._db_fixture = None
757
 
        if os.environ.get('LP_TEST_INSTANCE'):
758
 
            cls._db_template_fixture.tearDown()
759
 
            cls._db_template_fixture = None
 
759
        cls._db_template_fixture.tearDown()
 
760
        cls._db_template_fixture = None
760
761
 
761
762
    @classmethod
762
763
    @profiled
912
913
    @classmethod
913
914
    @profiled
914
915
    def _check_and_reset(cls):
915
 
        """Raise an exception if the Librarian has been killed, else reset."""
 
916
        """Raise an exception if the Librarian has been killed.
 
917
        Reset the storage unless this has been disabled.
 
918
        """
916
919
        try:
917
920
            f = urlopen(config.librarian.download_url)
918
921
            f.read()
923
926
                    "LibrarianLayer.reveal() where possible, and ensure "
924
927
                    "the Librarian is restarted if it absolutely must be "
925
928
                    "shutdown: " + str(e))
926
 
        else:
927
 
            cls.librarian_fixture.reset()
 
929
        cls.librarian_fixture.clear()
928
930
 
929
931
    @classmethod
930
932
    @profiled
1208
1210
        # This should not happen here, it should be caught by the
1209
1211
        # testTearDown() method. If it does, something very nasty
1210
1212
        # happened.
1211
 
        if getSecurityPolicy() != LaunchpadPermissiveSecurityPolicy:
 
1213
        if getSecurityPolicy() != PermissiveSecurityPolicy:
1212
1214
            raise LayerInvariantError(
1213
 
                "Previous test removed the LaunchpadPermissiveSecurityPolicy."
1214
 
                )
 
1215
                "Previous test removed the PermissiveSecurityPolicy.")
1215
1216
 
1216
1217
        # execute_zcml_for_scripts() sets up an interaction for the
1217
1218
        # anonymous user. A previous script may have changed or removed
1228
1229
                "Component architecture not loaded or totally screwed")
1229
1230
        # Make sure that a test that changed the security policy, reset it
1230
1231
        # back to its default value.
1231
 
        if getSecurityPolicy() != LaunchpadPermissiveSecurityPolicy:
 
1232
        if getSecurityPolicy() != PermissiveSecurityPolicy:
1232
1233
            raise LayerInvariantError(
1233
 
                "This test removed the LaunchpadPermissiveSecurityPolicy and "
1234
 
                "didn't restore it.")
 
1234
                "This test removed the PermissiveSecurityPolicy and didn't "
 
1235
                "restore it.")
1235
1236
        logout()
1236
1237
 
1237
1238
 
1350
1351
    @profiled
1351
1352
    def testSetUp(cls):
1352
1353
        # Connect Storm
1353
 
        reconnect_stores(reset=True)
 
1354
        reconnect_stores()
1354
1355
 
1355
1356
    @classmethod
1356
1357
    @profiled
1357
1358
    def testTearDown(cls):
1358
1359
        getUtility(IOpenLaunchBag).clear()
1359
1360
 
1360
 
        endInteraction()
 
1361
        # If tests forget to logout, we can do it for them.
 
1362
        if is_logged_in():
 
1363
            logout()
1361
1364
 
1362
1365
        # Disconnect Storm so it doesn't get in the way of database resets
1363
1366
        disconnect_stores()
1379
1382
        OpStats.resetStats()
1380
1383
 
1381
1384
        # Connect Storm
1382
 
        reconnect_stores(reset=True)
 
1385
        reconnect_stores()
1383
1386
 
1384
1387
    @classmethod
1385
1388
    @profiled
1386
1389
    def testTearDown(cls):
1387
1390
        getUtility(IOpenLaunchBag).clear()
1388
1391
 
1389
 
        endInteraction()
 
1392
        # If tests forget to logout, we can do it for them.
 
1393
        if is_logged_in():
 
1394
            logout()
1390
1395
 
1391
1396
        # Reset any statistics
1392
1397
        from canonical.launchpad.webapp.opstats import OpStats
1444
1449
    def testSetUp(cls):
1445
1450
        # LaunchpadZopelessLayer takes care of reconnecting the stores
1446
1451
        if not LaunchpadZopelessLayer.isSetUp:
1447
 
            reconnect_stores(reset=True)
 
1452
            reconnect_stores()
1448
1453
 
1449
1454
    @classmethod
1450
1455
    @profiled
1451
1456
    def testTearDown(cls):
1452
1457
        disconnect_stores()
1453
1458
 
 
1459
    @classmethod
 
1460
    @profiled
 
1461
    def switchDbConfig(cls, database_config_section):
 
1462
        reconnect_stores(database_config_section=database_config_section)
 
1463
 
1454
1464
 
1455
1465
class LaunchpadScriptLayer(ZopelessLayer, LaunchpadLayer):
1456
1466
    """Testing layer for scripts using the main Launchpad database adapter"""
1477
1487
    def testSetUp(cls):
1478
1488
        # LaunchpadZopelessLayer takes care of reconnecting the stores
1479
1489
        if not LaunchpadZopelessLayer.isSetUp:
1480
 
            reconnect_stores(reset=True)
 
1490
            reconnect_stores()
1481
1491
 
1482
1492
    @classmethod
1483
1493
    @profiled
1484
1494
    def testTearDown(cls):
1485
1495
        disconnect_stores()
1486
1496
 
 
1497
    @classmethod
 
1498
    @profiled
 
1499
    def switchDbConfig(cls, database_config_section):
 
1500
        reconnect_stores(database_config_section=database_config_section)
 
1501
 
1487
1502
 
1488
1503
class LaunchpadTestSetup(PgTestSetup):
1489
1504
    template = 'launchpad_ftest_template'
1490
1505
    dbuser = 'launchpad'
1491
 
    host = 'localhost'
1492
1506
 
1493
1507
 
1494
1508
class LaunchpadZopelessLayer(LaunchpadScriptLayer):
1497
1511
    """
1498
1512
 
1499
1513
    isSetUp = False
1500
 
    txn = transaction
 
1514
    txn = ZopelessTransactionManager
1501
1515
 
1502
1516
    @classmethod
1503
1517
    @profiled
1512
1526
    @classmethod
1513
1527
    @profiled
1514
1528
    def testSetUp(cls):
1515
 
        dbconfig.override(isolation_level='read_committed')
1516
 
        # XXX wgrant 2011-09-24 bug=29744: initZopeless used to do this.
1517
 
        # Tests that still need it should eventually set this directly,
1518
 
        # so the whole layer is not polluted.
1519
 
        set_immediate_mail_delivery(True)
 
1529
        if ZopelessTransactionManager._installed is not None:
 
1530
            raise LayerIsolationError(
 
1531
                "Last test using Zopeless failed to tearDown correctly")
 
1532
        initZopeless()
1520
1533
 
1521
1534
        # Connect Storm
1522
1535
        reconnect_stores()
1524
1537
    @classmethod
1525
1538
    @profiled
1526
1539
    def testTearDown(cls):
1527
 
        dbconfig.reset()
 
1540
        ZopelessTransactionManager.uninstall()
 
1541
        if ZopelessTransactionManager._installed is not None:
 
1542
            raise LayerInvariantError(
 
1543
                "Failed to uninstall ZopelessTransactionManager")
1528
1544
        # LaunchpadScriptLayer will disconnect the stores for us.
1529
1545
 
1530
 
        # XXX wgrant 2011-09-24 bug=29744: uninstall used to do this.
1531
 
        # Tests that still need immediate delivery should eventually do
1532
 
        # this directly.
1533
 
        set_immediate_mail_delivery(False)
1534
 
 
1535
1546
    @classmethod
1536
1547
    @profiled
1537
1548
    def commit(cls):
1545
1556
    @classmethod
1546
1557
    @profiled
1547
1558
    def switchDbUser(cls, dbuser):
1548
 
        # DEPRECATED: use switch_dbuser directly.
1549
 
        switch_dbuser(dbuser)
 
1559
        LaunchpadZopelessLayer.alterConnection(dbuser=dbuser)
 
1560
 
 
1561
    @classmethod
 
1562
    @profiled
 
1563
    def alterConnection(cls, **kw):
 
1564
        """Reset the connection, and reopen the connection by calling
 
1565
        initZopeless with the given keyword arguments.
 
1566
        """
 
1567
        ZopelessTransactionManager.uninstall()
 
1568
        initZopeless(**kw)
1550
1569
 
1551
1570
 
1552
1571
class ExperimentalLaunchpadZopelessLayer(LaunchpadZopelessLayer):
1727
1746
    smtp_controller = None
1728
1747
 
1729
1748
    @classmethod
1730
 
    def setConfig(cls):
 
1749
    def _setConfig(cls):
1731
1750
        """Stash a config for use."""
1732
1751
        cls.appserver_config = CanonicalConfig(
1733
1752
            BaseLayer.appserver_config_name, 'runlaunchpad')
1734
1753
 
1735
1754
    @classmethod
1736
1755
    def setUp(cls):
1737
 
        cls.setConfig()
 
1756
        cls._setConfig()
1738
1757
        cls.startSMTPServer()
1739
1758
        cls.startAppServer()
1740
1759
 
1760
1779
 
1761
1780
    @classmethod
1762
1781
    @profiled
1763
 
    def startAppServer(cls, run_name='run'):
 
1782
    def startAppServer(cls):
1764
1783
        """Start the app server if it hasn't already been started."""
1765
1784
        if cls.appserver is not None:
1766
1785
            raise LayerInvariantError('App server already running')
1767
1786
        cls._cleanUpStaleAppServer()
1768
 
        cls._runAppServer(run_name)
 
1787
        cls._runAppServer()
1769
1788
        cls._waitUntilAppServerIsReady()
1770
1789
 
1771
1790
    @classmethod
1857
1876
            pidfile.remove_pidfile('launchpad', cls.appserver_config)
1858
1877
 
1859
1878
    @classmethod
1860
 
    def _runAppServer(cls, run_name):
 
1879
    def _runAppServer(cls):
1861
1880
        """Start the app server using runlaunchpad.py"""
 
1881
        # The app server will not start at all if the database hasn't been
 
1882
        # correctly patched. The app server will make exactly this check,
 
1883
        # doing it here makes the error more obvious.
 
1884
        confirm_dbrevision_on_startup()
1862
1885
        _config = cls.appserver_config
1863
1886
        cmd = [
1864
 
            os.path.join(_config.root, 'bin', run_name),
 
1887
            os.path.join(_config.root, 'bin', 'run'),
1865
1888
            '-C', 'configs/%s/launchpad.conf' % _config.instance_name]
1866
1889
        environ = dict(os.environ)
1867
1890
        environ['LPCONFIG'] = _config.instance_name
1870
1893
            env=environ, cwd=_config.root)
1871
1894
 
1872
1895
    @classmethod
1873
 
    def appserver_root_url(cls):
1874
 
        return cls.appserver_config.vhost.mainsite.rooturl
1875
 
 
1876
 
    @classmethod
1877
1896
    def _waitUntilAppServerIsReady(cls):
1878
1897
        """Wait until the app server accepts connection."""
1879
1898
        assert cls.appserver is not None, "App server isn't started."
1880
 
        root_url = cls.appserver_root_url()
 
1899
        root_url = cls.appserver_config.vhost.mainsite.rooturl
1881
1900
        until = datetime.datetime.now() + WAIT_INTERVAL
1882
1901
        while until > datetime.datetime.now():
1883
1902
            try:
1987
2006
 
1988
2007
 
1989
2008
class YUITestLayer(FunctionalLayer):
1990
 
    """The layer for all YUITests cases."""
1991
 
 
1992
 
 
1993
 
class YUIAppServerLayer(MemcachedLayer):
1994
 
    """The layer for all YUIAppServer test cases."""
1995
 
 
1996
 
    @classmethod
1997
 
    @profiled
1998
 
    def setUp(cls):
1999
 
        LayerProcessController.setConfig()
2000
 
        LayerProcessController.startAppServer('run-testapp')
2001
 
 
2002
 
    @classmethod
2003
 
    @profiled
2004
 
    def tearDown(cls):
2005
 
        LayerProcessController.stopAppServer()
2006
 
 
2007
 
    @classmethod
2008
 
    @profiled
2009
 
    def testSetUp(cls):
2010
 
        LaunchpadLayer.resetSessionDb()
 
2009
    """The base class for all YUITests cases."""