14538.2.49
by Curtis Hovey
Updated copyright. |
1 |
# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
|
9240.4.10
by William Grant
Split and move the tests. |
2 |
# GNU Affero General Public License version 3 (see the file LICENSE).
|
3 |
||
4 |
from datetime import datetime |
|
5 |
import os |
|
6 |
from StringIO import StringIO |
|
7 |
import subprocess |
|
8 |
||
9 |
from zope.component import getUtility |
|
10 |
||
14606.2.1
by William Grant
Create lp.services.librarianserver, moving librarian.apachelogparser into it. And move the smoketest from canonical.librarian to lp.services.librarian. |
11 |
from lp.services.apachelogparser.base import ( |
12 |
get_method_and_path, |
|
13 |
parse_file, |
|
14 |
)
|
|
15 |
from lp.services.librarian.interfaces import ILibraryFileAliasSet |
|
16 |
from lp.services.librarianserver.apachelogparser import get_library_file_id |
|
17 |
from lp.services.log.logger import BufferLogger |
|
14600.1.2
by Curtis Hovey
Updated callsites to import from lp.testing, where the code has been for years. |
18 |
from lp.testing import ( |
11403.1.4
by Henning Eggers
Reformatted imports using format-imports script r32. |
19 |
ANONYMOUS, |
20 |
login, |
|
14606.2.1
by William Grant
Create lp.services.librarianserver, moving librarian.apachelogparser into it. And move the smoketest from canonical.librarian to lp.services.librarian. |
21 |
TestCase, |
11403.1.4
by Henning Eggers
Reformatted imports using format-imports script r32. |
22 |
)
|
14604.1.1
by Curtis Hovey
Separate test-authoring classes from test-running classes. |
23 |
from lp.testing.layers import ( |
11403.1.4
by Henning Eggers
Reformatted imports using format-imports script r32. |
24 |
DatabaseFunctionalLayer, |
25 |
ZopelessLayer, |
|
26 |
)
|
|
9240.4.10
by William Grant
Split and move the tests. |
27 |
|
28 |
||
29 |
here = os.path.dirname(__file__) |
|
30 |
||
31 |
||
32 |
class TestRequestParsing(TestCase): |
|
33 |
"""Test parsing the request part of an apache log line."""
|
|
34 |
||
35 |
def assertMethodAndFileIDAreCorrect(self, request): |
|
36 |
method, path = get_method_and_path(request) |
|
9240.4.12
by William Grant
Rename get_lfa_download_key to get_library_file_id, to be more understandable. |
37 |
file_id = get_library_file_id(path) |
9240.4.10
by William Grant
Split and move the tests. |
38 |
self.assertEqual(method, 'GET') |
39 |
self.assertEqual(file_id, '8196569') |
|
40 |
||
41 |
def test_return_value(self): |
|
42 |
request = 'GET /8196569/mediumubuntulogo.png HTTP/1.1' |
|
43 |
self.assertMethodAndFileIDAreCorrect(request) |
|
44 |
||
45 |
def test_return_value_for_http_path(self): |
|
46 |
request = ('GET http://launchpadlibrarian.net/8196569/' |
|
47 |
'mediumubuntulogo.png HTTP/1.1') |
|
48 |
self.assertMethodAndFileIDAreCorrect(request) |
|
49 |
||
50 |
def test_extra_slashes_are_ignored(self): |
|
51 |
request = 'GET http://launchpadlibrarian.net//8196569//foo HTTP/1.1' |
|
52 |
self.assertMethodAndFileIDAreCorrect(request) |
|
53 |
||
54 |
request = 'GET //8196569//foo HTTP/1.1' |
|
55 |
self.assertMethodAndFileIDAreCorrect(request) |
|
56 |
||
10072.1.1
by Guilherme Salgado
Fix the bug by normalizing white spaces in the request string |
57 |
def test_multiple_consecutive_white_spaces(self): |
58 |
# Some request strings might have multiple consecutive white spaces,
|
|
59 |
# but they're parsed just like if they didn't have the extra spaces.
|
|
60 |
request = 'GET /8196569/mediumubuntulogo.png HTTP/1.1' |
|
61 |
self.assertMethodAndFileIDAreCorrect(request) |
|
62 |
||
9240.4.10
by William Grant
Split and move the tests. |
63 |
def test_return_value_for_https_path(self): |
64 |
request = ('GET https://launchpadlibrarian.net/8196569/' |
|
65 |
'mediumubuntulogo.png HTTP/1.1') |
|
66 |
self.assertMethodAndFileIDAreCorrect(request) |
|
67 |
||
68 |
def test_return_value_for_request_missing_http_version(self): |
|
69 |
# HTTP 1.0 requests might omit the HTTP version so we must cope with
|
|
70 |
# them.
|
|
71 |
request = 'GET https://launchpadlibrarian.net/8196569/foo.png' |
|
72 |
self.assertMethodAndFileIDAreCorrect(request) |
|
73 |
||
74 |
def test_requests_for_paths_that_are_not_of_an_lfa_return_none(self): |
|
75 |
request = 'GET https://launchpadlibrarian.net/ HTTP/1.1' |
|
76 |
self.assertEqual( |
|
9240.4.12
by William Grant
Rename get_lfa_download_key to get_library_file_id, to be more understandable. |
77 |
get_library_file_id(get_method_and_path(request)[1]), None) |
9240.4.10
by William Grant
Split and move the tests. |
78 |
|
79 |
request = 'GET /robots.txt HTTP/1.1' |
|
80 |
self.assertEqual( |
|
9240.4.12
by William Grant
Rename get_lfa_download_key to get_library_file_id, to be more understandable. |
81 |
get_library_file_id(get_method_and_path(request)[1]), None) |
9240.4.10
by William Grant
Split and move the tests. |
82 |
|
83 |
request = 'GET /@@person HTTP/1.1' |
|
84 |
self.assertEqual( |
|
9240.4.12
by William Grant
Rename get_lfa_download_key to get_library_file_id, to be more understandable. |
85 |
get_library_file_id(get_method_and_path(request)[1]), None) |
9240.4.10
by William Grant
Split and move the tests. |
86 |
|
87 |
||
88 |
class TestLibrarianLogFileParsing(TestCase): |
|
89 |
"""Test the parsing of librarian log files."""
|
|
90 |
||
91 |
layer = ZopelessLayer |
|
92 |
||
93 |
def setUp(self): |
|
94 |
TestCase.setUp(self) |
|
95 |
self.logger = BufferLogger() |
|
96 |
||
97 |
def test_request_to_lfa_is_parsed(self): |
|
98 |
fd = StringIO( |
|
99 |
'69.233.136.42 - - [13/Jun/2008:14:55:22 +0100] "GET '
|
|
100 |
'/15018215/ul_logo_64x64.png HTTP/1.1" 200 2261 '
|
|
101 |
'"https://launchpad.net/~ubuntulite/+archive" "Mozilla"') |
|
11461.1.2
by Benji York
make logparser_max_parsed_lines respected across all log files, not |
102 |
downloads, parsed_bytes, ignored = parse_file( |
9240.4.10
by William Grant
Split and move the tests. |
103 |
fd, start_position=0, logger=self.logger, |
9240.4.12
by William Grant
Rename get_lfa_download_key to get_library_file_id, to be more understandable. |
104 |
get_download_key=get_library_file_id) |
11021.2.4
by Michael Nelson
Updated with extra logging for number of lines parsed and download counts created. |
105 |
self.assertEqual( |
12070.1.44
by Tim Penhey
Fix more change fallout in the tests. |
106 |
self.logger.getLogBuffer().strip(), |
107 |
'INFO Parsed 1 lines resulting in 1 download stats.') |
|
9240.4.10
by William Grant
Split and move the tests. |
108 |
|
109 |
date = datetime(2008, 6, 13) |
|
11021.2.4
by Michael Nelson
Updated with extra logging for number of lines parsed and download counts created. |
110 |
self.assertEqual(downloads, |
9240.4.10
by William Grant
Split and move the tests. |
111 |
{'15018215': {datetime(2008, 6, 13): {'US': 1}}}) |
112 |
||
113 |
self.assertEqual(parsed_bytes, fd.tell()) |
|
114 |
||
115 |
def test_request_to_non_lfa_is_ignored(self): |
|
116 |
# A request to a path which doesn't map to a LibraryFileAlias (e.g.
|
|
117 |
# '/') is ignored.
|
|
118 |
fd = StringIO( |
|
119 |
'69.233.136.42 - - [13/Jun/2008:14:55:22 +0100] "GET / HTTP/1.1" '
|
|
120 |
'200 2261 "https://launchpad.net/~ubuntulite/+archive" "Mozilla"') |
|
11461.1.2
by Benji York
make logparser_max_parsed_lines respected across all log files, not |
121 |
downloads, parsed_bytes, ignored = parse_file( |
9240.4.10
by William Grant
Split and move the tests. |
122 |
fd, start_position=0, logger=self.logger, |
9240.4.12
by William Grant
Rename get_lfa_download_key to get_library_file_id, to be more understandable. |
123 |
get_download_key=get_library_file_id) |
11021.2.4
by Michael Nelson
Updated with extra logging for number of lines parsed and download counts created. |
124 |
self.assertEqual( |
12070.1.44
by Tim Penhey
Fix more change fallout in the tests. |
125 |
self.logger.getLogBuffer().strip(), |
126 |
'INFO Parsed 1 lines resulting in 0 download stats.') |
|
9240.4.10
by William Grant
Split and move the tests. |
127 |
self.assertEqual(downloads, {}) |
128 |
self.assertEqual(parsed_bytes, fd.tell()) |
|
129 |
||
130 |
||
131 |
class TestScriptRunning(TestCase): |
|
132 |
"""Run parse-librarian-apache-access-logs.py and test its outcome."""
|
|
133 |
||
134 |
layer = DatabaseFunctionalLayer |
|
135 |
||
136 |
def test_script_run(self): |
|
137 |
# Before we run the script, the LibraryFileAliases with id 1, 2 and 3
|
|
138 |
# will have download counts set to 0. After the script's run, each of
|
|
139 |
# them will have their download counts set to 1, matching the sample
|
|
140 |
# log files we use for this test:
|
|
141 |
# scripts/tests/apache-log-files-for-sampledata.
|
|
142 |
login(ANONYMOUS) |
|
143 |
libraryfile_set = getUtility(ILibraryFileAliasSet) |
|
144 |
self.assertEqual(libraryfile_set[1].hits, 0) |
|
145 |
self.assertEqual(libraryfile_set[2].hits, 0) |
|
146 |
self.assertEqual(libraryfile_set[3].hits, 0) |
|
147 |
||
148 |
process = subprocess.Popen( |
|
149 |
'cronscripts/parse-librarian-apache-access-logs.py', shell=True, |
|
150 |
stdin=subprocess.PIPE, stdout=subprocess.PIPE, |
|
151 |
stderr=subprocess.PIPE) |
|
152 |
(out, err) = process.communicate() |
|
153 |
self.assertEqual( |
|
154 |
process.returncode, 0, "stdout:%s, stderr:%s" % (out, err)) |
|
155 |
||
156 |
# Must commit because the changes were done in another transaction.
|
|
157 |
import transaction |
|
158 |
transaction.commit() |
|
159 |
self.assertEqual(libraryfile_set[1].hits, 1) |
|
160 |
self.assertEqual(libraryfile_set[2].hits, 1) |
|
161 |
self.assertEqual(libraryfile_set[3].hits, 1) |