1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
|
# Copyright 2009-2010 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Archive uploader utilities."""
__metaclass__ = type
__all__ = [
'DpkgSourceError',
'extract_dpkg_source',
're_taint_free',
're_isadeb',
're_issource',
're_is_component_orig_tar_ext',
're_no_epoch',
're_no_revision',
're_valid_version',
're_valid_pkg_name',
're_changes_file_name',
're_extract_src_version',
'get_source_file_extension',
'determine_binary_file_type',
'determine_source_file_type',
'prefix_multi_line_string',
'safe_fix_maintainer',
'ParseMaintError',
]
import email.Header
import re
import signal
import subprocess
from lp.services.encoding import (
ascii_smash,
guess as guess_encoding,
)
from lp.soyuz.enums import BinaryPackageFileType
class DpkgSourceError(Exception):
_fmt = "Unable to unpack source package (%(result)s): %(output)s"
def __init__(self, command, output, result):
super(DpkgSourceError, self).__init__(
self._fmt % {
"output": output, "result": result, "command": command})
self.output = output
self.result = result
self.command = command
re_taint_free = re.compile(r"^[-+~/\.\w]+$")
re_isadeb = re.compile(r"(.+?)_(.+?)_(.+)\.(u?d?deb)$")
source_file_exts = [
'orig(?:-.+)?\.tar\.(?:gz|bz2|xz)', 'diff.gz',
'(?:debian\.)?tar\.(?:gz|bz2|xz)', 'dsc']
re_issource = re.compile(
r"([^_]+)_(.+?)\.(%s)" % "|".join(ext for ext in source_file_exts))
re_is_component_orig_tar_ext = re.compile(r"^orig-(.+).tar.(?:gz|bz2|xz)$")
re_is_orig_tar_ext = re.compile(r"^orig.tar.(?:gz|bz2|xz)$")
re_is_debian_tar_ext = re.compile(r"^debian.tar.(?:gz|bz2|xz)$")
re_is_native_tar_ext = re.compile(r"^tar.(?:gz|bz2|xz)$")
re_no_epoch = re.compile(r"^\d+\:")
re_no_revision = re.compile(r"-[^-]+$")
re_valid_version = re.compile(r"^([0-9]+:)?[0-9A-Za-z\.\-\+~:]+$")
re_valid_pkg_name = re.compile(r"^[\dA-Za-z][\dA-Za-z\+\-\.]+$")
re_changes_file_name = re.compile(r"([^_]+)_([^_]+)_([^\.]+).changes")
re_extract_src_version = re.compile(r"(\S+)\s*\((.*)\)")
re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\>]+)\>")
def get_source_file_extension(filename):
"""Get the extension part of a source file name."""
match = re_issource.match(filename)
if match is None:
return None
return match.group(3)
def determine_source_file_type(filename):
"""Determine the SourcePackageFileType of the given filename."""
# Avoid circular imports.
from lp.registry.interfaces.sourcepackage import SourcePackageFileType
extension = get_source_file_extension(filename)
if extension is None:
return None
elif extension == "dsc":
return SourcePackageFileType.DSC
elif extension == "diff.gz":
return SourcePackageFileType.DIFF
elif re_is_orig_tar_ext.match(extension):
return SourcePackageFileType.ORIG_TARBALL
elif re_is_component_orig_tar_ext.match(extension):
return SourcePackageFileType.COMPONENT_ORIG_TARBALL
elif re_is_debian_tar_ext.match(extension):
return SourcePackageFileType.DEBIAN_TARBALL
elif re_is_native_tar_ext.match(extension):
return SourcePackageFileType.NATIVE_TARBALL
else:
return None
def determine_binary_file_type(filename):
"""Determine the BinaryPackageFileType of the given filename."""
if filename.endswith(".deb"):
return BinaryPackageFileType.DEB
elif filename.endswith(".udeb"):
return BinaryPackageFileType.UDEB
elif filename.endswith(".ddeb"):
return BinaryPackageFileType.DDEB
else:
return None
def prefix_multi_line_string(str, prefix, include_blank_lines=0):
"""Utility function to split an input string and prefix,
Each line with a token or tag. Can be used for quoting text etc.
"""
out = ""
for line in str.split('\n'):
line = line.strip()
if line or include_blank_lines:
out += "%s%s\n" % (prefix, line)
# Strip trailing new line
if out:
out = out[:-1]
return out
def extract_component_from_section(section, default_component="main"):
component = ""
if section.find("/") != -1:
component, section = section.split("/")
else:
component = default_component
return (section, component)
def force_to_utf8(s):
"""Forces a string to UTF-8.
If the string isn't already UTF-8, it's assumed to be ISO-8859-1.
"""
try:
unicode(s, 'utf-8')
return s
except UnicodeError:
latin1_s = unicode(s, 'iso8859-1')
return latin1_s.encode('utf-8')
def rfc2047_encode(s):
"""Encodes a (header) string per RFC2047 if necessary.
If the string is neither ASCII nor UTF-8, it's assumed to be ISO-8859-1.
"""
if not s:
return ''
try:
s.decode('us-ascii')
#encodings.ascii.Codec().decode(s)
return s
except UnicodeError:
pass
try:
s.decode('utf8')
#encodings.utf_8.Codec().decode(s)
h = email.Header.Header(s, 'utf-8', 998)
return str(h)
except UnicodeError:
h = email.Header.Header(s, 'iso-8859-1', 998)
return str(h)
class ParseMaintError(Exception):
"""Exception raised for errors in parsing a maintainer field.
Attributes:
message -- explanation of the error
"""
def __init__(self, message):
Exception.__init__(self)
self.args = (message, )
self.message = message
def fix_maintainer(maintainer, field_name="Maintainer"):
"""Parses a Maintainer or Changed-By field and returns:
(1) an RFC822 compatible version,
(2) an RFC2047 compatible version,
(3) the name
(4) the email
The name is forced to UTF-8 for both (1) and (3). If the name field
contains '.' or ',', (1) and (2) are switched to 'email (name)' format.
"""
maintainer = maintainer.strip()
if not maintainer:
return ('', '', '', '')
if maintainer.find("<") == -1:
email = maintainer
name = ""
elif (maintainer[0] == "<" and maintainer[-1:] == ">"):
email = maintainer[1:-1]
name = ""
else:
m = re_parse_maintainer.match(maintainer)
if not m:
raise ParseMaintError(
"%s: doesn't parse as a valid %s field."
% (maintainer, field_name))
name = m.group(1)
email = m.group(2)
# Just in case the maintainer ended up with nested angles; check...
while email.startswith("<"):
email = email[1:]
# Get an RFC2047 compliant version of the name
rfc2047_name = rfc2047_encode(name)
# Force the name to be UTF-8
name = force_to_utf8(name)
# If the maintainer's name contains a full stop then the whole field will
# not work directly as an email address due to a misfeature in the syntax
# specified in RFC822; see Debian policy 5.6.2 (Maintainer field syntax)
# for details.
if name.find(',') != -1 or name.find('.') != -1:
rfc822_maint = "%s (%s)" % (email, name)
rfc2047_maint = "%s (%s)" % (email, rfc2047_name)
else:
rfc822_maint = "%s <%s>" % (name, email)
rfc2047_maint = "%s <%s>" % (rfc2047_name, email)
if email.find("@") == -1 and email.find("buildd_") != 0:
raise ParseMaintError(
"%s: no @ found in email address part." % maintainer)
return (rfc822_maint, rfc2047_maint, name, email)
def safe_fix_maintainer(content, fieldname):
"""Wrapper for fix_maintainer() to handle unicode and string argument.
It verifies the content type and transform it in a unicode with guess()
before call ascii_smash(). Then we can safely call fix_maintainer().
"""
if type(content) != unicode:
content = guess_encoding(content)
content = ascii_smash(content)
return fix_maintainer(content, fieldname)
def extract_dpkg_source(dsc_filepath, target):
"""Extract a source package by dsc file path.
:param dsc_filepath: Path of the DSC file
:param target: Target directory
"""
def subprocess_setup():
# Python installs a SIGPIPE handler by default. This is usually not
# what non-Python subprocesses expect.
# http://www.chiark.greenend.org.uk/ucgi/~cjwatson/ \
# blosxom/2009-07-02-python-sigpipe.html
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
args = ["dpkg-source", "-sn", "-x", dsc_filepath]
dpkg_source = subprocess.Popen(
args, stdout=subprocess.PIPE, cwd=target, stderr=subprocess.PIPE,
preexec_fn=subprocess_setup)
output, unused = dpkg_source.communicate()
result = dpkg_source.wait()
if result != 0:
dpkg_output = prefix_multi_line_string(output, " ")
raise DpkgSourceError(result=result, output=dpkg_output, command=args)
|