1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
|
# Copyright 2009 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
from cStringIO import StringIO
import textwrap
import unittest
from urllib2 import (
HTTPError,
URLError,
)
import transaction
from canonical.database.sqlbase import block_implicit_flushes
from lp.services.config import config
from lp.services.database.lpstorm import ISlaveStore
from lp.services.librarian import client as client_module
from lp.services.librarian.client import (
LibrarianClient,
LibrarianServerError,
RestrictedLibrarianClient,
)
from lp.services.librarian.interfaces.client import UploadFailed
from lp.services.librarian.model import LibraryFileAlias
from lp.services.webapp.dbpolicy import SlaveDatabasePolicy
from lp.testing.layers import (
DatabaseLayer,
LaunchpadFunctionalLayer,
)
class InstrumentedLibrarianClient(LibrarianClient):
def __init__(self, *args, **kwargs):
super(InstrumentedLibrarianClient, self).__init__(*args, **kwargs)
self.check_error_calls = 0
sentDatabaseName = False
def _sendHeader(self, name, value):
if name == 'Database-Name':
self.sentDatabaseName = True
return LibrarianClient._sendHeader(self, name, value)
called_getURLForDownload = False
def _getURLForDownload(self, aliasID):
self.called_getURLForDownload = True
return LibrarianClient._getURLForDownload(self, aliasID)
def _checkError(self):
self.check_error_calls += 1
super(InstrumentedLibrarianClient, self)._checkError()
def make_mock_file(error, max_raise):
"""Return a surrogate for client._File.
The surrogate function raises error when called for the first
max_raise times.
"""
file_status = {
'error': error,
'max_raise': max_raise,
'num_calls': 0,
}
def mock_file(url_file, url):
if file_status['num_calls'] < file_status['max_raise']:
file_status['num_calls'] += 1
raise file_status['error']
return 'This is a fake file object'
return mock_file
class LibrarianClientTestCase(unittest.TestCase):
layer = LaunchpadFunctionalLayer
def test_addFileSendsDatabaseName(self):
# addFile should send the Database-Name header.
client = InstrumentedLibrarianClient()
id1 = client.addFile(
'sample.txt', 6, StringIO('sample'), 'text/plain')
self.failUnless(client.sentDatabaseName,
"Database-Name header not sent by addFile")
def test_remoteAddFileDoesntSendDatabaseName(self):
# remoteAddFile should send the Database-Name header as well.
client = InstrumentedLibrarianClient()
# Because the remoteAddFile call commits to the database in a
# different process, we need to explicitly tell the DatabaseLayer to
# fully tear down and set up the database.
DatabaseLayer.force_dirty_database()
id1 = client.remoteAddFile('sample.txt', 6, StringIO('sample'),
'text/plain')
self.failUnless(client.sentDatabaseName,
"Database-Name header not sent by remoteAddFile")
def test_clientWrongDatabase(self):
# If the client is using the wrong database, the server should refuse
# the upload, causing LibrarianClient to raise UploadFailed.
client = LibrarianClient()
# Force the client to mis-report its database
client._getDatabaseName = lambda cur: 'wrong_database'
try:
client.addFile('sample.txt', 6, StringIO('sample'), 'text/plain')
except UploadFailed, e:
msg = e.args[0]
self.failUnless(
msg.startswith('Server said: 400 Wrong database'),
'Unexpected UploadFailed error: ' + msg)
else:
self.fail("UploadFailed not raised")
def test_addFile_uses_master(self):
# addFile is a write operation, so it should always use the
# master store, even if the slave is the default. Close the
# slave store and try to add a file, verifying that the master
# is used.
client = LibrarianClient()
ISlaveStore(LibraryFileAlias).close()
with SlaveDatabasePolicy():
alias_id = client.addFile(
'sample.txt', 6, StringIO('sample'), 'text/plain')
transaction.commit()
f = client.getFileByAlias(alias_id)
self.assertEqual(f.read(), 'sample')
def test_addFile_no_response_check_at_end_headers_for_empty_file(self):
# When addFile() sends the request header, it checks if the
# server responded with an error response after sending each
# header line. It does _not_ do this check when it sends the
# empty line following the headers.
client = InstrumentedLibrarianClient()
client.addFile(
'sample.txt', 0, StringIO(''), 'text/plain',
allow_zero_length=True)
# addFile() calls _sendHeader() three times and _sendLine()
# twice, but it does not check if the server responded
# in the second call.
self.assertEqual(4, client.check_error_calls)
def test_addFile_response_check_at_end_headers_for_non_empty_file(self):
# When addFile() sends the request header, it checks if the
# server responded with an error response after sending each
# header line. It does _not_ do this check when it sends the
# empty line following the headers.
client = InstrumentedLibrarianClient()
client.addFile('sample.txt', 4, StringIO('1234'), 'text/plain')
# addFile() calls _sendHeader() three times and _sendLine()
# twice.
self.assertEqual(5, client.check_error_calls)
def test__getURLForDownload(self):
# This protected method is used by getFileByAlias. It is supposed to
# use the internal host and port rather than the external, proxied
# host and port. This is to provide relief for our own issues with the
# problems reported in bug 317482.
#
# (Set up:)
client = LibrarianClient()
alias_id = client.addFile(
'sample.txt', 6, StringIO('sample'), 'text/plain')
config.push(
'test config',
textwrap.dedent('''\
[librarian]
download_host: example.org
download_port: 1234
'''))
try:
# (Test:)
# The LibrarianClient should use the download_host and
# download_port.
expected_host = 'http://example.org:1234/'
download_url = client._getURLForDownload(alias_id)
self.failUnless(download_url.startswith(expected_host),
'expected %s to start with %s' % (download_url,
expected_host))
# If the alias has been deleted, _getURLForDownload returns None.
lfa = LibraryFileAlias.get(alias_id)
lfa.content = None
call = block_implicit_flushes( # Prevent a ProgrammingError
LibrarianClient._getURLForDownload)
self.assertEqual(call(client, alias_id), None)
finally:
# (Tear down:)
config.pop('test config')
def test_restricted_getURLForDownload(self):
# The RestrictedLibrarianClient should use the
# restricted_download_host and restricted_download_port, but is
# otherwise identical to the behavior of the LibrarianClient discussed
# and demonstrated above.
#
# (Set up:)
client = RestrictedLibrarianClient()
alias_id = client.addFile(
'sample.txt', 6, StringIO('sample'), 'text/plain')
config.push(
'test config',
textwrap.dedent('''\
[librarian]
restricted_download_host: example.com
restricted_download_port: 5678
'''))
try:
# (Test:)
# The LibrarianClient should use the download_host and
# download_port.
expected_host = 'http://example.com:5678/'
download_url = client._getURLForDownload(alias_id)
self.failUnless(download_url.startswith(expected_host),
'expected %s to start with %s' % (download_url,
expected_host))
# If the alias has been deleted, _getURLForDownload returns None.
lfa = LibraryFileAlias.get(alias_id)
lfa.content = None
call = block_implicit_flushes( # Prevent a ProgrammingError
RestrictedLibrarianClient._getURLForDownload)
self.assertEqual(call(client, alias_id), None)
finally:
# (Tear down:)
config.pop('test config')
def test_getFileByAlias(self):
# This method should use _getURLForDownload to download the file.
# We use the InstrumentedLibrarianClient to show that it is consulted.
#
# (Set up:)
client = InstrumentedLibrarianClient()
alias_id = client.addFile(
'sample.txt', 6, StringIO('sample'), 'text/plain')
transaction.commit() # Make sure the file is in the "remote" database
self.failIf(client.called_getURLForDownload)
# (Test:)
f = client.getFileByAlias(alias_id)
self.assertEqual(f.read(), 'sample')
self.failUnless(client.called_getURLForDownload)
def test_getFileByAliasLookupError(self):
# The Librarian server can return a 404 HTTPError;
# LibrarienClient.getFileByAlias() returns a LookupError in
# this case.
_File = client_module._File
client_module._File = make_mock_file(
HTTPError('http://fake.url/', 404, 'Forced error', None, None), 1)
client = InstrumentedLibrarianClient()
alias_id = client.addFile(
'sample.txt', 6, StringIO('sample'), 'text/plain')
transaction.commit()
self.assertRaises(LookupError, client.getFileByAlias, alias_id)
client_module._File = _File
def test_getFileByAliasLibrarianLongServerError(self):
# The Librarian server can return a 500 HTTPError.
# LibrarienClient.getFileByAlias() returns a LibrarianServerError
# if the server returns this error for a longer time than given
# by the parameter timeout.
_File = client_module._File
client_module._File = make_mock_file(
HTTPError('http://fake.url/', 500, 'Forced error', None, None), 2)
client = InstrumentedLibrarianClient()
alias_id = client.addFile(
'sample.txt', 6, StringIO('sample'), 'text/plain')
transaction.commit()
self.assertRaises(
LibrarianServerError, client.getFileByAlias, alias_id, 1)
client_module._File = make_mock_file(
URLError('Connection refused'), 2)
client = InstrumentedLibrarianClient()
alias_id = client.addFile(
'sample.txt', 6, StringIO('sample'), 'text/plain')
transaction.commit()
self.assertRaises(
LibrarianServerError, client.getFileByAlias, alias_id, 1)
client_module._File = _File
def test_getFileByAliasLibrarianShortServerError(self):
# The Librarian server can return a 500 HTTPError;
# LibrarienClient.getFileByAlias() returns a LibrarianServerError
# in this case.
_File = client_module._File
client_module._File = make_mock_file(
HTTPError('http://fake.url/', 500, 'Forced error', None, None), 1)
client = InstrumentedLibrarianClient()
alias_id = client.addFile(
'sample.txt', 6, StringIO('sample'), 'text/plain')
transaction.commit()
self.assertEqual(
client.getFileByAlias(alias_id), 'This is a fake file object', 3)
client_module._File = make_mock_file(
URLError('Connection refused'), 1)
client = InstrumentedLibrarianClient()
alias_id = client.addFile(
'sample.txt', 6, StringIO('sample'), 'text/plain')
transaction.commit()
self.assertEqual(
client.getFileByAlias(alias_id), 'This is a fake file object', 3)
client_module._File = _File
|