~launchpad-pqm/launchpad/devel

14538.2.49 by Curtis Hovey
Updated copyright.
1
# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
9240.4.10 by William Grant
Split and move the tests.
2
# GNU Affero General Public License version 3 (see the file LICENSE).
3
4
from datetime import datetime
5
import os
6
from StringIO import StringIO
7
import subprocess
8
9
from zope.component import getUtility
10
14606.2.1 by William Grant
Create lp.services.librarianserver, moving librarian.apachelogparser into it. And move the smoketest from canonical.librarian to lp.services.librarian.
11
from lp.services.apachelogparser.base import (
12
    get_method_and_path,
13
    parse_file,
14
    )
15
from lp.services.librarian.interfaces import ILibraryFileAliasSet
16
from lp.services.librarianserver.apachelogparser import get_library_file_id
17
from lp.services.log.logger import BufferLogger
14600.1.2 by Curtis Hovey
Updated callsites to import from lp.testing, where the code has been for years.
18
from lp.testing import (
11403.1.4 by Henning Eggers
Reformatted imports using format-imports script r32.
19
    ANONYMOUS,
20
    login,
14606.2.1 by William Grant
Create lp.services.librarianserver, moving librarian.apachelogparser into it. And move the smoketest from canonical.librarian to lp.services.librarian.
21
    TestCase,
11403.1.4 by Henning Eggers
Reformatted imports using format-imports script r32.
22
    )
14604.1.1 by Curtis Hovey
Separate test-authoring classes from test-running classes.
23
from lp.testing.layers import (
11403.1.4 by Henning Eggers
Reformatted imports using format-imports script r32.
24
    DatabaseFunctionalLayer,
25
    ZopelessLayer,
26
    )
9240.4.10 by William Grant
Split and move the tests.
27
28
29
here = os.path.dirname(__file__)
30
31
32
class TestRequestParsing(TestCase):
33
    """Test parsing the request part of an apache log line."""
34
35
    def assertMethodAndFileIDAreCorrect(self, request):
36
        method, path = get_method_and_path(request)
9240.4.12 by William Grant
Rename get_lfa_download_key to get_library_file_id, to be more understandable.
37
        file_id = get_library_file_id(path)
9240.4.10 by William Grant
Split and move the tests.
38
        self.assertEqual(method, 'GET')
39
        self.assertEqual(file_id, '8196569')
40
41
    def test_return_value(self):
42
        request = 'GET /8196569/mediumubuntulogo.png HTTP/1.1'
43
        self.assertMethodAndFileIDAreCorrect(request)
44
45
    def test_return_value_for_http_path(self):
46
        request = ('GET http://launchpadlibrarian.net/8196569/'
47
                   'mediumubuntulogo.png HTTP/1.1')
48
        self.assertMethodAndFileIDAreCorrect(request)
49
50
    def test_extra_slashes_are_ignored(self):
51
        request = 'GET http://launchpadlibrarian.net//8196569//foo HTTP/1.1'
52
        self.assertMethodAndFileIDAreCorrect(request)
53
54
        request = 'GET //8196569//foo HTTP/1.1'
55
        self.assertMethodAndFileIDAreCorrect(request)
56
10072.1.1 by Guilherme Salgado
Fix the bug by normalizing white spaces in the request string
57
    def test_multiple_consecutive_white_spaces(self):
58
        # Some request strings might have multiple consecutive white spaces,
59
        # but they're parsed just like if they didn't have the extra spaces.
60
        request = 'GET /8196569/mediumubuntulogo.png  HTTP/1.1'
61
        self.assertMethodAndFileIDAreCorrect(request)
62
9240.4.10 by William Grant
Split and move the tests.
63
    def test_return_value_for_https_path(self):
64
        request = ('GET https://launchpadlibrarian.net/8196569/'
65
                   'mediumubuntulogo.png HTTP/1.1')
66
        self.assertMethodAndFileIDAreCorrect(request)
67
68
    def test_return_value_for_request_missing_http_version(self):
69
        # HTTP 1.0 requests might omit the HTTP version so we must cope with
70
        # them.
71
        request = 'GET https://launchpadlibrarian.net/8196569/foo.png'
72
        self.assertMethodAndFileIDAreCorrect(request)
73
74
    def test_requests_for_paths_that_are_not_of_an_lfa_return_none(self):
75
        request = 'GET https://launchpadlibrarian.net/ HTTP/1.1'
76
        self.assertEqual(
9240.4.12 by William Grant
Rename get_lfa_download_key to get_library_file_id, to be more understandable.
77
            get_library_file_id(get_method_and_path(request)[1]), None)
9240.4.10 by William Grant
Split and move the tests.
78
79
        request = 'GET /robots.txt HTTP/1.1'
80
        self.assertEqual(
9240.4.12 by William Grant
Rename get_lfa_download_key to get_library_file_id, to be more understandable.
81
            get_library_file_id(get_method_and_path(request)[1]), None)
9240.4.10 by William Grant
Split and move the tests.
82
83
        request = 'GET /@@person HTTP/1.1'
84
        self.assertEqual(
9240.4.12 by William Grant
Rename get_lfa_download_key to get_library_file_id, to be more understandable.
85
            get_library_file_id(get_method_and_path(request)[1]), None)
9240.4.10 by William Grant
Split and move the tests.
86
87
88
class TestLibrarianLogFileParsing(TestCase):
89
    """Test the parsing of librarian log files."""
90
91
    layer = ZopelessLayer
92
93
    def setUp(self):
94
        TestCase.setUp(self)
95
        self.logger = BufferLogger()
96
97
    def test_request_to_lfa_is_parsed(self):
98
        fd = StringIO(
99
            '69.233.136.42 - - [13/Jun/2008:14:55:22 +0100] "GET '
100
            '/15018215/ul_logo_64x64.png HTTP/1.1" 200 2261 '
101
            '"https://launchpad.net/~ubuntulite/+archive" "Mozilla"')
11461.1.2 by Benji York
make logparser_max_parsed_lines respected across all log files, not
102
        downloads, parsed_bytes, ignored = parse_file(
9240.4.10 by William Grant
Split and move the tests.
103
            fd, start_position=0, logger=self.logger,
9240.4.12 by William Grant
Rename get_lfa_download_key to get_library_file_id, to be more understandable.
104
            get_download_key=get_library_file_id)
11021.2.4 by Michael Nelson
Updated with extra logging for number of lines parsed and download counts created.
105
        self.assertEqual(
12070.1.44 by Tim Penhey
Fix more change fallout in the tests.
106
            self.logger.getLogBuffer().strip(),
107
            'INFO Parsed 1 lines resulting in 1 download stats.')
9240.4.10 by William Grant
Split and move the tests.
108
109
        date = datetime(2008, 6, 13)
11021.2.4 by Michael Nelson
Updated with extra logging for number of lines parsed and download counts created.
110
        self.assertEqual(downloads,
9240.4.10 by William Grant
Split and move the tests.
111
            {'15018215': {datetime(2008, 6, 13): {'US': 1}}})
112
113
        self.assertEqual(parsed_bytes, fd.tell())
114
115
    def test_request_to_non_lfa_is_ignored(self):
116
        # A request to a path which doesn't map to a LibraryFileAlias (e.g.
117
        # '/') is ignored.
118
        fd = StringIO(
119
            '69.233.136.42 - - [13/Jun/2008:14:55:22 +0100] "GET / HTTP/1.1" '
120
            '200 2261 "https://launchpad.net/~ubuntulite/+archive" "Mozilla"')
11461.1.2 by Benji York
make logparser_max_parsed_lines respected across all log files, not
121
        downloads, parsed_bytes, ignored = parse_file(
9240.4.10 by William Grant
Split and move the tests.
122
            fd, start_position=0, logger=self.logger,
9240.4.12 by William Grant
Rename get_lfa_download_key to get_library_file_id, to be more understandable.
123
            get_download_key=get_library_file_id)
11021.2.4 by Michael Nelson
Updated with extra logging for number of lines parsed and download counts created.
124
        self.assertEqual(
12070.1.44 by Tim Penhey
Fix more change fallout in the tests.
125
            self.logger.getLogBuffer().strip(),
126
            'INFO Parsed 1 lines resulting in 0 download stats.')
9240.4.10 by William Grant
Split and move the tests.
127
        self.assertEqual(downloads, {})
128
        self.assertEqual(parsed_bytes, fd.tell())
129
130
131
class TestScriptRunning(TestCase):
132
    """Run parse-librarian-apache-access-logs.py and test its outcome."""
133
134
    layer = DatabaseFunctionalLayer
135
136
    def test_script_run(self):
137
        # Before we run the script, the LibraryFileAliases with id 1, 2 and 3
138
        # will have download counts set to 0.  After the script's run, each of
139
        # them will have their download counts set to 1, matching the sample
140
        # log files we use for this test:
141
        # scripts/tests/apache-log-files-for-sampledata.
142
        login(ANONYMOUS)
143
        libraryfile_set = getUtility(ILibraryFileAliasSet)
144
        self.assertEqual(libraryfile_set[1].hits, 0)
145
        self.assertEqual(libraryfile_set[2].hits, 0)
146
        self.assertEqual(libraryfile_set[3].hits, 0)
147
148
        process = subprocess.Popen(
149
            'cronscripts/parse-librarian-apache-access-logs.py', shell=True,
150
            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
151
            stderr=subprocess.PIPE)
152
        (out, err) = process.communicate()
153
        self.assertEqual(
154
            process.returncode, 0, "stdout:%s, stderr:%s" % (out, err))
155
156
        # Must commit because the changes were done in another transaction.
157
        import transaction
158
        transaction.commit()
159
        self.assertEqual(libraryfile_set[1].hits, 1)
160
        self.assertEqual(libraryfile_set[2].hits, 1)
161
        self.assertEqual(libraryfile_set[3].hits, 1)