1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
|
# Copyright 2009 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Helper utilities."""
__metaclass__ = type
__all__ = [
'apply_for_list',
'collect_archive_message_ids',
'create_list',
'ensure_addresses_are_disabled',
'ensure_addresses_are_enabled',
'ensure_membership',
'ensure_nonmembership',
'get_size',
'pending_hold_ids',
'print_mailman_hold',
'review_list',
'run_mailman',
'subscribe',
'unsubscribe',
]
import datetime
import errno
import os
import pickle
import re
from subprocess import (
PIPE,
Popen,
)
import time
# This is where the Mailman command line scripts live.
import Mailman
from Mailman import mm_cfg
from Mailman.Errors import NotAMemberError
from Mailman.MailList import MailList
from Mailman.MemberAdaptor import (
BYUSER,
ENABLED,
)
from Mailman.Utils import list_names
import transaction
from zope.component import getUtility
from zope.security.proxy import removeSecurityProxy
from lp.registry.interfaces.mailinglist import IMailingListSet
from lp.registry.interfaces.person import IPersonSet
from lp.registry.tests import mailinglists_helper
from lp.services.mailman.testing.layers import MailmanLayer
from lp.testing import celebrity_logged_in
from lp.testing.browser import Browser
from lp.testing.factory import LaunchpadObjectFactory
MAILMAN_PKGDIR = os.path.dirname(Mailman.__file__)
MAILMAN_BINDIR = os.path.join(os.path.dirname(MAILMAN_PKGDIR), 'bin')
SPACE = ' '
MAILING_LIST_CHECK_INTERVAL = datetime.timedelta(seconds=10)
SECONDS_TO_SNOOZE = 0.1
def get_size(path):
"""Return the size of a file, or -1 if it doesn't exist."""
try:
return os.stat(path).st_size
except OSError, error:
if error.errno == errno.ENOENT:
# Return -1 when the file does not exist, so it always
# compares less than an existing but empty file.
return -1
# Some other error occurred.
raise
def review_list(list_name, status='approve'):
"""Helper for approving a mailing list."""
result = MailmanLayer.xmlrpc_watcher.wait_for_create(list_name)
if result is not None:
# The watch timed out.
print result
return None
with celebrity_logged_in('admin'):
mailing_list = getUtility(IMailingListSet).get(list_name)
return mailing_list
def create_list(team_name):
"""Do everything you need to do to make the team's list live."""
displayname = SPACE.join(
word.capitalize() for word in team_name.split('-'))
browser = Browser('no-priv@canonical.com:test')
# Create the team.
browser.open('%s/people/+newteam' % MailmanLayer.appserver_root_url())
browser.getControl(name='field.name').value = team_name
browser.getControl('Display Name').value = displayname
browser.getControl(name='field.subscriptionpolicy').displayValue = [
'Open Team']
browser.getControl('Create').click()
# Create the mailing list.
browser.getLink('Create a mailing list').click()
browser.getControl('Create new Mailing List').click()
mailing_list = review_list(team_name)
# pylint: disable-msg=F0401
assert team_name in list_names(), (
'Mailing list was not created: %s (found: %s)' %
(team_name, list_names()))
result = ensure_membership(team_name, 'archive@mail-archive.dev')
if result is not None:
return result
return mailing_list
def run_mailman(*args):
"""Run a Mailman script."""
proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=MAILMAN_BINDIR)
stdout, stderr = proc.communicate()
assert len(stderr) == 0, stderr
return stdout
def subscribe(first_name, team_name, use_alt_address=False):
"""Do everything you need to subscribe a person to a mailing list."""
# Create the person if she does not already exist, and join her to the
# team.
with celebrity_logged_in('admin'):
person_set = getUtility(IPersonSet)
person = person_set.getByName(first_name.lower())
if person is None:
person = LaunchpadObjectFactory().makePersonByName(first_name)
team = getUtility(IPersonSet).getByName(team_name)
person.join(team)
# Subscribe her to the list.
mailing_list = getUtility(IMailingListSet).get(team_name)
if use_alt_address:
alternative_email = mailinglists_helper.get_alternative_email(
person)
mailing_list.subscribe(person, alternative_email)
else:
mailing_list.subscribe(person)
transaction.commit()
return ensure_membership(team_name, person)
def unsubscribe(first_name, team_name):
"""Unsubscribe the named person from the team's mailing list."""
with celebrity_logged_in('admin'):
person_set = getUtility(IPersonSet)
person = person_set.getByName(first_name.lower())
assert person is not None, 'No such person: %s' % first_name
mailing_list = getUtility(IMailingListSet).get(team_name)
mailing_list.unsubscribe(person)
transaction.commit()
# Unsubscribing does not make the person a non-member, but it
# does disable all their addresses.
addresses = [
removeSecurityProxy(email).email
for email in person.validatedemails
]
addresses.append(removeSecurityProxy(person.preferredemail).email)
return ensure_addresses_are_disabled(team_name, *addresses)
def pending_hold_ids(list_name):
"""Return the set of pending held messages in Mailman for the list.
We do it this way in order to be totally safe, so that there's no
possibility of leaving a locked list floating around. doctest doesn't
always do the right thing.
"""
# The list must be locked to make this query.
mailing_list = MailList(list_name)
try:
return mailing_list.GetHeldMessageIds()
finally:
mailing_list.Unlock()
def print_mailman_hold(list_name, hold_id):
"""Print the held message as Mailman sees it."""
# The list must be locked to make this query.
mailing_list = MailList(list_name)
try:
data = mailing_list.GetRecord(hold_id)
finally:
mailing_list.Unlock()
held_file_name = data[4]
path = os.path.join(mm_cfg.DATA_DIR, held_file_name)
file_object = open(path)
try:
message = pickle.load(file_object)
finally:
file_object.close()
print message.as_string()
def collect_archive_message_ids(team_name):
"""Collect all the X-Message-Id values in the team's archived messages."""
mhonarc_path = os.path.join(mm_cfg.VAR_PREFIX, 'mhonarc', 'itest-one')
message_ids = []
# Unfortunately, there's nothing we can wait on to know whether the
# archiver has run yet or not, because the archive runner does not log
# messages when it completes.
archived_files = []
for count in range(3):
try:
archived_files = [file_name
for file_name in os.listdir(mhonarc_path)
if file_name.endswith('.html')]
except OSError, error:
if error.errno != errno.ENOENT:
raise
# Sleep and try again.
if len(archived_files) > 0:
break
time.sleep(0.5)
for html_file in archived_files:
archive_file = open(os.path.join(mhonarc_path, html_file))
try:
data = archive_file.read()
finally:
archive_file.close()
for line in data.splitlines():
if line.startswith('<!DOCTYPE'):
break
mo = re.match('<!--X-Message-Id:\s*(?P<id>[\S]+)', line, re.I)
if mo:
message_ids.append(mo.group('id'))
break
return sorted(message_ids)
def apply_for_list(browser, team_name):
"""Like mailinglists_helper.apply_for_list() but with the right rooturl.
"""
mailinglists_helper.apply_for_list(
browser, team_name, MailmanLayer.appserver_root_url(ensureSlash=True))
def _membership_test(team_name, people, predicate):
"""Test membership via the predicate.
:param team_name: the name of the team/mailing list to test
:type team_name: string
:param people: the sequence of IPersons to check. All validated emails
from all persons are collected and checked for membership.
:type people: sequence of IPersons
:param predicate: A function taking two arguments. The first argument is
the sent of member addresses found in the Mailman MailList
data structure. The second argument is the set of validated email
addresses for all the `people`. The function should return a boolean
indicating whether the condition being tested is satisfied or not.
:type predicate: function
:return: the string 'Timed out' if the predicate never succeeded, or None
if it did.
:rtype: string or None
"""
member_addresses = set()
for person in people:
if isinstance(person, basestring):
member_addresses.add(person)
else:
for email in person.validatedemails:
member_addresses.add(removeSecurityProxy(email).email)
# Also add the preferred address.
preferred = removeSecurityProxy(person.preferredemail).email
member_addresses.add(preferred)
assert len(member_addresses) > 0, 'No valid addresses found'
mailing_list = MailList(team_name, lock=False)
until = datetime.datetime.now() + MAILING_LIST_CHECK_INTERVAL
while True:
members = set(
mailing_list.getMemberCPAddresses(mailing_list.getMembers()))
if predicate(members, member_addresses):
# Every address in the arguments was a member. Return None
# on success, so that the doctest doesn't need an extra line to
# match the output.
return None
# The predicate test failed. See if we timed out.
if datetime.datetime.now() > until:
return 'Timed out'
time.sleep(SECONDS_TO_SNOOZE)
# Reload the mailing list data and go around again.
mailing_list.Load()
def ensure_membership(team_name, *people):
"""Ensure that all the addresses are members of the mailing list."""
def all_are_members(list_members, wanted_members):
return list_members.issuperset(wanted_members)
return _membership_test(team_name, people, all_are_members)
def ensure_nonmembership(team_name, *people):
"""Ensure that none of the addresses are members of the mailing list."""
def none_are_members(list_members, unwanted_members):
# The intersection of the two sets is empty.
return len(list_members & unwanted_members) == 0
return _membership_test(team_name, people, none_are_members)
def _ensure_addresses_are_in_state(team_name, state, addresses):
"""Ensure that addresses are in the specified state."""
mailing_list = MailList(team_name, lock=False)
until = datetime.datetime.now() + MAILING_LIST_CHECK_INTERVAL
while True:
for address in addresses:
try:
if mailing_list.getDeliveryStatus(address) != state:
break
except NotAMemberError:
# The address is not a member, so it can't be in the state.
break
else:
# All addresses are in the specified state. Return None on
# success, so that the doctest doesn't need an extra line to match
# the output.
return None
if datetime.datetime.now() > until:
return 'Timed out'
time.sleep(SECONDS_TO_SNOOZE)
# Reload the mailing list data and go around again.
mailing_list.Load()
def ensure_addresses_are_enabled(team_name, *addresses):
"""Ensure that addresses are subscribed and enabled."""
_ensure_addresses_are_in_state(team_name, ENABLED, addresses)
def ensure_addresses_are_disabled(team_name, *addresses):
"""Ensure that addresses are subscribed but disabled."""
# Use BYUSER because that's the non-enabled state that the implementation
# uses to represent an unsubscribed team member.
_ensure_addresses_are_in_state(team_name, BYUSER, addresses)
|