1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
|
# Copyright 2010 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Test dscfile.py"""
__metaclass__ = type
import os
from canonical.testing.layers import LaunchpadZopelessLayer
from lp.archiveuploader.dscfile import (
cleanup_unpacked_dir,
DSCFile,
find_changelog,
find_copyright,
format_to_file_checker_map,
unpack_source,
)
from lp.archiveuploader.nascentuploadfile import UploadError
from lp.archiveuploader.tests import datadir
from lp.archiveuploader.uploadpolicy import BuildDaemonUploadPolicy
from lp.registry.interfaces.sourcepackage import SourcePackageFileType
from lp.services.log.logger import BufferLogger, DevNullLogger
from lp.soyuz.enums import SourcePackageFormat
from lp.testing import (
TestCase,
TestCaseWithFactory,
)
ORIG_TARBALL = SourcePackageFileType.ORIG_TARBALL
DEBIAN_TARBALL = SourcePackageFileType.DEBIAN_TARBALL
NATIVE_TARBALL = SourcePackageFileType.NATIVE_TARBALL
DIFF = SourcePackageFileType.DIFF
class TestDscFile(TestCase):
def setUp(self):
super(TestDscFile, self).setUp()
self.tmpdir = self.makeTemporaryDirectory()
self.dir_path = os.path.join(self.tmpdir, "foo", "debian")
os.makedirs(self.dir_path)
self.copyright_path = os.path.join(self.dir_path, "copyright")
self.changelog_path = os.path.join(self.dir_path, "changelog")
def testBadDebianCopyright(self):
"""Test that a symlink as debian/copyright will fail.
This is a security check, to make sure its not possible to use a
dangling symlink in an attempt to try and access files on the system
processing the source packages."""
os.symlink("/etc/passwd", self.copyright_path)
error = self.assertRaises(
UploadError, find_copyright, self.tmpdir, DevNullLogger())
self.assertEqual(
error.args[0], "Symbolic link for debian/copyright not allowed")
def testGoodDebianCopyright(self):
"""Test that a proper copyright file will be accepted"""
copyright = "copyright for dummies"
file = open(self.copyright_path, "w")
file.write(copyright)
file.close()
self.assertEquals(
copyright, find_copyright(self.tmpdir, DevNullLogger()))
def testBadDebianChangelog(self):
"""Test that a symlink as debian/changelog will fail.
This is a security check, to make sure its not possible to use a
dangling symlink in an attempt to try and access files on the system
processing the source packages."""
os.symlink("/etc/passwd", self.changelog_path)
error = self.assertRaises(
UploadError, find_changelog, self.tmpdir, DevNullLogger())
self.assertEqual(
error.args[0], "Symbolic link for debian/changelog not allowed")
def testGoodDebianChangelog(self):
"""Test that a proper changelog file will be accepted"""
changelog = "changelog for dummies"
file = open(self.changelog_path, "w")
file.write(changelog)
file.close()
self.assertEquals(
changelog, find_changelog(self.tmpdir, DevNullLogger()))
def testOversizedFile(self):
"""Test that a file larger than 10MiB will fail.
This check exists to prevent a possible denial of service attack
against launchpad by overloaded the database or librarian with massive
changelog and copyright files. 10MiB was set as a sane lower limit
which is incredibly unlikely to be hit by normal files in the
archive"""
dev_zero = open("/dev/zero", "r")
ten_MiB = 2**20 * 10
empty_file = dev_zero.read(ten_MiB + 1)
dev_zero.close()
file = open(self.changelog_path, "w")
file.write(empty_file)
file.close()
error = self.assertRaises(
UploadError, find_changelog, self.tmpdir, DevNullLogger())
self.assertEqual(
error.args[0], "debian/changelog file too large, 10MiB max")
class TestDscFileLibrarian(TestCaseWithFactory):
"""Tests for DscFile that may use the Librarian."""
layer = LaunchpadZopelessLayer
def getDscFile(self, name):
dsc_path = datadir(os.path.join('suite', name, name + '.dsc'))
class Changes:
architectures = ['source']
logger = BufferLogger()
policy = BuildDaemonUploadPolicy()
policy.distroseries = self.factory.makeDistroSeries()
policy.archive = self.factory.makeArchive()
policy.distro = policy.distroseries.distribution
return DSCFile(dsc_path, 'digest', 0, 'main/editors',
'priority', 'package', 'version', Changes, policy, logger)
def test_ReadOnlyCWD(self):
"""Processing a file should work when cwd is read-only."""
tempdir = self.useTempDir()
os.chmod(tempdir, 0555)
try:
dsc_file = self.getDscFile('bar_1.0-1')
list(dsc_file.verify())
finally:
os.chmod(tempdir, 0755)
class BaseTestSourceFileVerification(TestCase):
def assertErrorsForFiles(self, expected, files, components={},
bzip2_count=0, xz_count=0):
"""Check problems with the given set of files for the given format.
:param expected: a list of expected errors, as strings.
:param format: the `SourcePackageFormat` to check against.
:param files: a dict mapping `SourcePackageFileType`s to counts.
:param components: a dict mapping orig component tarball components
to counts.
:param bzip2_count: number of files using bzip2 compression.
:param xz_count: number of files using xz compression.
"""
full_files = {
NATIVE_TARBALL: 0,
ORIG_TARBALL: 0,
DIFF: 0,
DEBIAN_TARBALL: 0,
}
full_files.update(files)
self.assertEquals(
expected,
[str(e) for e in format_to_file_checker_map[self.format](
'foo_1.dsc', full_files, components, bzip2_count, xz_count)])
def assertFilesOK(self, files, components={}, bzip2_count=0, xz_count=0):
"""Check that the given set of files is OK for the given format.
:param format: the `SourcePackageFormat` to check against.
:param files: a dict mapping `SourcePackageFileType`s to counts.
:param components: a dict mapping orig component tarball components
to counts.
:param bzip2_count: number of files using bzip2 compression.
:param xz_count: number of files using xz compression.
"""
self.assertErrorsForFiles([], files, components, bzip2_count, xz_count)
class Test10SourceFormatVerification(BaseTestSourceFileVerification):
format = SourcePackageFormat.FORMAT_1_0
wrong_files_error = ('foo_1.dsc: must have exactly one tar.gz, or an '
'orig.tar.gz and diff.gz')
bzip2_error = 'foo_1.dsc: is format 1.0 but uses bzip2 compression.'
xz_error = 'foo_1.dsc: is format 1.0 but uses xz compression.'
def testFormat10Debian(self):
# A 1.0 source can contain an original tarball and a Debian diff
self.assertFilesOK({ORIG_TARBALL: 1, DIFF: 1})
def testFormat10Native(self):
# A 1.0 source can contain a native tarball.
self.assertFilesOK({NATIVE_TARBALL: 1})
def testFormat10CannotHaveWrongFiles(self):
# A 1.0 source cannot have a combination of native and
# non-native files, and cannot have just one of the non-native
# files.
for combination in (
{DIFF: 1}, {ORIG_TARBALL: 1}, {ORIG_TARBALL: 1, DIFF: 1,
NATIVE_TARBALL: 1}):
self.assertErrorsForFiles([self.wrong_files_error], combination)
# A 1.0 source with component tarballs is invalid.
self.assertErrorsForFiles(
[self.wrong_files_error], {ORIG_TARBALL: 1, DIFF: 1}, {'foo': 1})
def testFormat10CannotUseBzip2(self):
# 1.0 sources cannot use bzip2 compression.
self.assertErrorsForFiles(
[self.bzip2_error], {NATIVE_TARBALL: 1}, {}, 1, 0)
def testFormat10CannotUseXz(self):
# 1.0 sources cannot use xz compression.
self.assertErrorsForFiles(
[self.xz_error], {NATIVE_TARBALL: 1}, {}, 0, 1)
class Test30QuiltSourceFormatVerification(BaseTestSourceFileVerification):
format = SourcePackageFormat.FORMAT_3_0_QUILT
wrong_files_error = ('foo_1.dsc: must have only an orig.tar.*, a '
'debian.tar.* and optionally orig-*.tar.*')
comp_conflict_error = 'foo_1.dsc: has more than one orig-bar.tar.*.'
def testFormat30Quilt(self):
# A 3.0 (quilt) source must contain an orig tarball and a debian
# tarball. It may also contain at most one component tarball for
# each component, and can use gzip, bzip2, or xz compression.
for components in ({}, {'foo': 1}, {'foo': 1, 'bar': 1}):
for bzip2_count in (0, 1):
for xz_count in (0, 1):
self.assertFilesOK(
{ORIG_TARBALL: 1, DEBIAN_TARBALL: 1}, components,
bzip2_count, xz_count)
def testFormat30QuiltCannotHaveConflictingComponentTarballs(self):
# Multiple conflicting tarballs for a single component are
# invalid.
self.assertErrorsForFiles(
[self.comp_conflict_error],
{ORIG_TARBALL: 1, DEBIAN_TARBALL: 1}, {'foo': 1, 'bar': 2})
def testFormat30QuiltCannotHaveWrongFiles(self):
# 3.0 (quilt) sources may not have a diff or native tarball.
for filetype in (DIFF, NATIVE_TARBALL):
self.assertErrorsForFiles(
[self.wrong_files_error],
{ORIG_TARBALL: 1, DEBIAN_TARBALL: 1, filetype: 1})
class Test30QuiltSourceFormatVerification(BaseTestSourceFileVerification):
format = SourcePackageFormat.FORMAT_3_0_NATIVE
wrong_files_error = 'foo_1.dsc: must have only a tar.*.'
def testFormat30Native(self):
# 3.0 (native) sources must contain just a native tarball. They
# may use gzip, bzip2, or xz compression.
for bzip2_count in (0, 1):
self.assertFilesOK({NATIVE_TARBALL: 1}, {},
bzip2_count, 0)
self.assertFilesOK({NATIVE_TARBALL: 1}, {}, 0, 1)
def testFormat30NativeCannotHaveWrongFiles(self):
# 3.0 (quilt) sources may not have a diff, Debian tarball, orig
# tarball, or any component tarballs.
for filetype in (DIFF, DEBIAN_TARBALL, ORIG_TARBALL):
self.assertErrorsForFiles(
[self.wrong_files_error], {NATIVE_TARBALL: 1, filetype: 1})
# A 3.0 (native) source with component tarballs is invalid.
self.assertErrorsForFiles(
[self.wrong_files_error], {NATIVE_TARBALL: 1}, {'foo': 1})
class UnpackedDirTests(TestCase):
"""Tests for unpack_source and cleanup_unpacked_dir."""
def test_unpack_source(self):
# unpack_source unpacks in a temporary directory and returns the
# path.
unpacked_dir = unpack_source(
datadir(os.path.join('suite', 'bar_1.0-1', 'bar_1.0-1.dsc')))
try:
self.assertEquals(["bar-1.0"], os.listdir(unpacked_dir))
self.assertContentEqual(
["THIS_IS_BAR", "debian"],
os.listdir(os.path.join(unpacked_dir, "bar-1.0")))
finally:
cleanup_unpacked_dir(unpacked_dir)
def test_cleanup(self):
# cleanup_dir removes the temporary directory and all files under it.
temp_dir = self.makeTemporaryDirectory()
unpacked_dir = os.path.join(temp_dir, "unpacked")
os.mkdir(unpacked_dir)
os.mkdir(os.path.join(unpacked_dir, "bar_1.0"))
cleanup_unpacked_dir(unpacked_dir)
self.assertFalse(os.path.exists(unpacked_dir))
def test_cleanup_invalid_mode(self):
# cleanup_dir can remove a directory even if the mode does
# not allow it.
temp_dir = self.makeTemporaryDirectory()
unpacked_dir = os.path.join(temp_dir, "unpacked")
os.mkdir(unpacked_dir)
bar_path = os.path.join(unpacked_dir, "bar_1.0")
os.mkdir(bar_path)
os.chmod(bar_path, 0600)
os.chmod(unpacked_dir, 0600)
cleanup_unpacked_dir(unpacked_dir)
self.assertFalse(os.path.exists(unpacked_dir))
|