1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
|
#!/usr/bin/python
#
# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
# pylint: disable-msg=C0103,W0403
from datetime import (
datetime,
timedelta,
)
import filecmp
import os
import tempfile
import pytz
from zope.component import getUtility
from lp.archivepublisher.config import getPubConfig
from lp.archivepublisher.htaccess import (
htpasswd_credentials_for_archive,
write_htaccess,
write_htpasswd,
)
from lp.registry.model.teammembership import TeamParticipation
from lp.services.config import config
from lp.services.database.lpstorm import IStore
from lp.services.mail.helpers import get_email_template
from lp.services.mail.mailwrapper import MailWrapper
from lp.services.mail.sendmail import (
format_address,
simple_sendmail,
)
from lp.services.scripts.base import LaunchpadCronScript
from lp.services.scripts.interfaces.scriptactivity import IScriptActivitySet
from lp.services.webapp import canonical_url
from lp.soyuz.enums import (
ArchiveStatus,
ArchiveSubscriberStatus,
)
from lp.soyuz.model.archiveauthtoken import ArchiveAuthToken
from lp.soyuz.model.archivesubscriber import ArchiveSubscriber
# These PPAs should never have their htaccess/pwd files touched.
BLACKLISTED_PPAS = {
'ubuntuone': ['ppa'],
}
class HtaccessTokenGenerator(LaunchpadCronScript):
"""Helper class for generating .htaccess files for private PPAs."""
blacklist = BLACKLISTED_PPAS
def add_my_options(self):
"""Add script command line options."""
self.parser.add_option(
"-n", "--dry-run", action="store_true",
dest="dryrun", default=False,
help="If set, no files are changed and no tokens are "
"deactivated.")
self.parser.add_option(
"-d", "--no-deactivation", action="store_true",
dest="no_deactivation", default=False,
help="If set, tokens are not deactivated.")
def ensureHtaccess(self, ppa):
"""Generate a .htaccess for `ppa`."""
if self.options.dryrun:
return
# The publisher Config object does not have an
# interface, so we need to remove the security wrapper.
pub_config = getPubConfig(ppa)
htaccess_filename = os.path.join(pub_config.htaccessroot, ".htaccess")
if not os.path.exists(htaccess_filename):
# It's not there, so create it.
if not os.path.exists(pub_config.htaccessroot):
os.makedirs(pub_config.htaccessroot)
write_htaccess(htaccess_filename, pub_config.htaccessroot)
self.logger.debug("Created .htaccess for %s" % ppa.displayname)
def generateHtpasswd(self, ppa, tokens):
"""Generate a htpasswd file for `ppa`s `tokens`.
:param ppa: The context PPA (an `IArchive`).
:param tokens: A iterable containing `IArchiveAuthToken`s.
:return: The filename of the htpasswd file that was generated.
"""
# Create a temporary file that will be a new .htpasswd.
pub_config = getPubConfig(ppa)
fd, temp_filename = tempfile.mkstemp(dir=pub_config.htaccessroot)
os.close(fd)
write_htpasswd(
temp_filename, htpasswd_credentials_for_archive(ppa, tokens))
return temp_filename
def replaceUpdatedHtpasswd(self, ppa, temp_htpasswd_file):
"""Compare the new and the old htpasswd and replace if changed.
:return: True if the file was replaced.
"""
if self.options.dryrun:
return False
# The publisher Config object does not have an
# interface, so we need to remove the security wrapper.
pub_config = getPubConfig(ppa)
htpasswd_filename = os.path.join(pub_config.htaccessroot, ".htpasswd")
if (not os.path.isfile(htpasswd_filename) or
not filecmp.cmp(htpasswd_filename, temp_htpasswd_file)):
# Atomically replace the old file or create a new file.
os.rename(temp_htpasswd_file, htpasswd_filename)
self.logger.debug("Replaced htpasswd for %s" % ppa.displayname)
return True
return False
def sendCancellationEmail(self, token):
"""Send an email to the person whose subscription was cancelled."""
send_to_person = token.person
ppa_name = token.archive.displayname
ppa_owner_url = canonical_url(token.archive.owner)
subject = "PPA access cancelled for %s" % ppa_name
template = get_email_template(
"ppa-subscription-cancelled.txt", app='soyuz')
assert not send_to_person.is_team, (
"Token.person is a team, it should always be individuals.")
if send_to_person.preferredemail is None:
# The person has no preferred email set, so we don't
# email them.
return
to_address = [send_to_person.preferredemail.email]
replacements = {
'recipient_name': send_to_person.displayname,
'ppa_name': ppa_name,
'ppa_owner_url': ppa_owner_url,
}
body = MailWrapper(72).format(
template % replacements, force_wrap=True)
from_address = format_address(
ppa_name,
config.canonical.noreply_from_address)
headers = {
'Sender': config.canonical.bounce_address,
}
simple_sendmail(from_address, to_address, subject, body, headers)
def deactivateTokens(self, send_email=False):
"""Deactivate tokens as necessary.
If a subscriber no longer has an active token for the PPA, we
deactivate it.
:param send_email: Whether to send a cancellation email to the owner
of the token. This defaults to False to speed up the test
suite.
:return: the set of ppas affected by token deactivations.
"""
# First we grab all the active tokens for which there is a
# matching current archive subscription for a team of which the
# token owner is a member.
store = IStore(ArchiveSubscriber)
valid_tokens = store.find(
ArchiveAuthToken,
ArchiveAuthToken.date_deactivated == None,
ArchiveAuthToken.archive_id == ArchiveSubscriber.archive_id,
ArchiveSubscriber.status == ArchiveSubscriberStatus.CURRENT,
ArchiveSubscriber.subscriber_id == TeamParticipation.teamID,
TeamParticipation.personID == ArchiveAuthToken.person_id)
# We can then evaluate the invalid tokens by the difference of
# all active tokens and valid tokens.
all_active_tokens = store.find(
ArchiveAuthToken,
ArchiveAuthToken.date_deactivated == None)
invalid_tokens = all_active_tokens.difference(valid_tokens)
# We note the ppas affected by these token deactivations so that
# we can later update their htpasswd files.
affected_ppas = set()
for token in invalid_tokens:
if send_email:
self.sendCancellationEmail(token)
token.deactivate()
affected_ppas.add(token.archive)
return affected_ppas
def expireSubscriptions(self):
"""Expire subscriptions as necessary.
If an `ArchiveSubscriber`'s date_expires has passed, then
set its status to EXPIRED.
"""
now = datetime.now(pytz.UTC)
store = IStore(ArchiveSubscriber)
newly_expired_subscriptions = store.find(
ArchiveSubscriber,
ArchiveSubscriber.status == ArchiveSubscriberStatus.CURRENT,
ArchiveSubscriber.date_expires != None,
ArchiveSubscriber.date_expires <= now)
subscription_names = [
subs.displayname for subs in newly_expired_subscriptions]
if subscription_names:
newly_expired_subscriptions.set(
status=ArchiveSubscriberStatus.EXPIRED)
self.logger.info(
"Expired subscriptions: %s" % ", ".join(subscription_names))
def getNewTokensSinceLastRun(self):
"""Return result set of new tokens created since the last run."""
store = IStore(ArchiveAuthToken)
# If we don't know when we last ran, we include all active
# tokens by default.
last_success = getUtility(IScriptActivitySet).getLastActivity(
'generate-ppa-htaccess')
extra_expr = []
if last_success:
# NTP is running on our servers and therefore we can assume
# only minimal skew, we include a fudge-factor of 1s so that
# even the minimal skew cannot demonstrate bug 627608.
last_script_start_with_skew = last_success.date_started - (
timedelta(seconds=1))
extra_expr = [
ArchiveAuthToken.date_created >= last_script_start_with_skew]
new_ppa_tokens = store.find(
ArchiveAuthToken,
ArchiveAuthToken.date_deactivated == None,
*extra_expr)
return new_ppa_tokens
def _getValidTokensForPPAs(self, ppas):
"""Returns a dict keyed by archive with all the tokens for each."""
affected_ppas_with_tokens = dict([
(ppa, []) for ppa in ppas])
store = IStore(ArchiveAuthToken)
affected_ppa_tokens = store.find(
ArchiveAuthToken,
ArchiveAuthToken.date_deactivated == None,
ArchiveAuthToken.archive_id.is_in(
[ppa.id for ppa in ppas]))
for token in affected_ppa_tokens:
affected_ppas_with_tokens[token.archive].append(token)
return affected_ppas_with_tokens
def getNewPrivatePPAs(self):
"""Return the recently created private PPAs."""
# Avoid circular import.
from lp.soyuz.model.archive import Archive
store = IStore(Archive)
# If we don't know when we last ran, we include all active
# tokens by default.
last_success = getUtility(IScriptActivitySet).getLastActivity(
'generate-ppa-htaccess')
extra_expr = []
if last_success:
extra_expr = [
Archive.date_created >= last_success.date_completed]
return store.find(
Archive, Archive._private == True, *extra_expr)
def main(self):
"""Script entry point."""
self.logger.info('Starting the PPA .htaccess generation')
self.expireSubscriptions()
affected_ppas = self.deactivateTokens(send_email=True)
# In addition to the ppas that are affected by deactivated
# tokens, we also want to include any ppas that have tokens
# created since the last time we ran.
for token in self.getNewTokensSinceLastRun():
affected_ppas.add(token.archive)
affected_ppas.update(self.getNewPrivatePPAs())
affected_ppas_with_tokens = {}
if affected_ppas:
affected_ppas_with_tokens = self._getValidTokensForPPAs(
affected_ppas)
for ppa, valid_tokens in affected_ppas_with_tokens.iteritems():
# If this PPA is blacklisted, do not touch it's htaccess/pwd
# files.
blacklisted_ppa_names_for_owner = self.blacklist.get(
ppa.owner.name, [])
if ppa.name in blacklisted_ppa_names_for_owner:
self.logger.info(
"Skipping htacess updates for blacklisted PPA "
" '%s' owned by %s.",
ppa.name,
ppa.owner.displayname)
continue
elif ppa.status == ArchiveStatus.DELETED or ppa.enabled is False:
self.logger.info(
"Skipping htacess updates for deleted or disabled PPA "
" '%s' owned by %s.",
ppa.name,
ppa.owner.displayname)
continue
self.ensureHtaccess(ppa)
temp_htpasswd = self.generateHtpasswd(ppa, valid_tokens)
if not self.replaceUpdatedHtpasswd(ppa, temp_htpasswd):
os.remove(temp_htpasswd)
if self.options.no_deactivation or self.options.dryrun:
self.logger.info('Dry run, so not committing transaction.')
self.txn.abort()
else:
self.logger.info('Committing transaction...')
self.txn.commit()
self.logger.info('Finished PPA .htaccess generation')
|