1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
|
#!/usr/bin/python
#
# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Migrate modules from the old LP directory structure to the new using
a control file and the exising mover script that Francis wrote.
"""
import errno
from optparse import OptionParser
import os
import re
from find import (
find_files,
find_matches,
)
from rename_module import (
bzr_add,
bzr_move_file,
bzr_remove_file,
rename_module,
update_references,
)
from rename_zcml import handle_zcml
from utils import (
log,
run,
spew,
)
MOVER = os.path.join(os.path.dirname(__file__), 'rename_module.py')
TLA_MAP = dict(
ans='answers',
app='app',
blu='blueprints',
bug='bugs',
cod='code',
reg='registry',
sha='shared',
soy='soyuz',
svc='services',
tes='testing',
tra='translations',
pkg='registry',
hdb='hardwaredb',
idy='services/identity',
ver='services/verification',
sta='services/statistics',
loo='services/looptuner',
fee='services/feeds',
)
RENAME_MAP = dict(
components='adapters',
database='model',
ftests='tests',
pagetests='stories',
)
OLD_TOP = 'lib/canonical/launchpad'
NEW_TOP = 'lib/lp'
APP_DIRECTORIES = [
'browser',
'doc',
'emailtemplates',
'interfaces',
'model',
'scripts',
'stories',
'templates',
'tests',
'browser/tests',
]
TEST_PATHS = set(('doc', 'tests', 'ftests', 'pagetests'))
# Ripped straight from GNU touch(1)
FLAGS = os.O_WRONLY | os.O_CREAT | os.O_NONBLOCK | os.O_NOCTTY
def parse_args():
"""Return a tuple of parser, option, and arguments."""
usage = """\
%prog [options] controlfile app_codes+
controlfile is the file containing the list of files to be moved. Each file
is prefixed with a TLA identifying the apps.
app_codes is a list of TLAs identifying the apps to migrate.
"""
parser = OptionParser(usage)
parser.add_option(
'--dryrun',
action='store_true', default=False, dest='dry_run',
help=("If this option is used actions will be printed "
"but not executed."))
parser.add_option(
'--no-move',
action='store_false', default=True, dest='move',
help="Don't actually move any files, just set up the app's tree.")
options, arguments = parser.parse_args()
return parser, options, arguments
def convert_ctl_data(data):
"""Return a dict of files, each keyed to an app."""
app_data = {}
for line in data:
try:
tla, fn = line.split()
except ValueError:
continue
if not tla in app_data:
app_data[tla] = []
app_data[tla].append(fn[2:])
return app_data
COLLIDED = []
def move_it(old_path, new_path):
"""Move a versioned file without colliding with another file."""
# Move the file and fix the imports. LBYL.
if os.path.exists(new_path):
if os.path.getsize(new_path) == 0:
# We must remove the file since bzr refuses to clobber existing
# files.
bzr_remove_file(new_path)
else:
log('COLLISION! target already exists: %s', new_path)
COLLIDED.append(new_path)
# Try to find an alternative. I seriously doubt we'll ever have
# more than two collisions.
for counter in range(10):
fn, ext = os.path.splitext(new_path)
new_target = fn + '_%d' % counter + ext
log(' new target: %s', new_target)
if not os.path.exists(new_target):
new_path = new_target
break
else:
raise AssertionError('Too many collisions: ' + new_path)
rename_module(old_path, new_path)
def make_tree(app):
"""Make the official tree structure."""
if not os.path.exists(NEW_TOP):
os.mkdir(NEW_TOP)
tld = os.path.join(NEW_TOP, TLA_MAP[app])
for directory in [''] + APP_DIRECTORIES:
d = os.path.join(tld, directory)
try:
os.mkdir(d)
bzr_add(d)
print "created", d
except OSError, e:
if e.errno == errno.EEXIST:
# The directory already exists, so assume that the __init__.py
# file also exists.
continue
else:
raise
else:
# Touch an empty __init__.py to make the thing a package.
init_file = os.path.join(d, '__init__.py')
fd = os.open(init_file, FLAGS, 0666)
os.close(fd)
bzr_add(init_file)
# Add the whole directory.
bzr_add(tld)
def file2module(module_file):
"""From a filename, return the python module name."""
start_path = 'lib' + os.path.sep
module_file, dummy = os.path.splitext(module_file)
module = module_file[len(start_path):].replace(os.path.sep, '.')
return module
def handle_script(old_path, new_path):
"""Move a script or directory and update references in cronscripts."""
parts = old_path.split(os.path.sep)
if (len(parts) - parts.index('scripts')) > 2:
# The script is a directory not a single-file module.
# Just get the directory portion and move everything at once.
old_path = os.path.join(*parts[:-1])
new_full_path = new_path
else:
# The script is a single-file module. Add the script name to the end
# of new_path.
new_full_path = os.path.join(new_path, parts[-1])
# Move the file or directory
bzr_move_file(old_path, new_path)
# Update references, but only in the cronscripts directory.
source_module = file2module(old_path)
target_module = file2module(new_full_path)
update_references(source_module, target_module)
update_helper_imports(old_path, new_full_path)
def map_filename(path):
"""Return the renamed file name."""
fn, dummy = os.path.splitext(path)
if fn.endswith('-pages'):
# Don't remap -pages doctests here.
return path
else:
return os.sep.join(RENAME_MAP.get(path_part, path_part)
for path_part in path.split(os.sep))
def handle_test(old_path, new_path):
"""Migrate tests."""
spew('handle_test(%s, %s)', old_path, new_path)
unsupported_dirs = [
'components',
'daemons',
'model',
'interfaces',
'mail',
'mailout',
'translationformat',
'utilities',
'validators',
'vocabularies',
'webapp',
'xmlrpc',
]
new_path = map_filename(new_path)
# Do target -pages.txt doctest remapping.
file_name, ext = os.path.splitext(new_path)
if file_name.endswith('-pages'):
new_path = file_name[:-6] + '-views' + ext
parts = new_path.split(os.sep)
index = parts.index('doc')
parts[index:index + 1] = ['browser', 'tests']
new_path = os.sep.join(parts)
if '/tests/' in new_path and '/browser/tests/' not in new_path:
# All unit tests except to browser unit tests move to the app
# tests dir.
new_path = os.sep.join(
path_part for path_part in new_path.split(os.sep)
if path_part not in unsupported_dirs)
# Create new_path's directory if it doesn't exist yet.
try:
test_dir, dummy = os.path.split(new_path)
os.makedirs(test_dir)
spew('created: %s', test_dir)
except OSError, error:
if error.errno != errno.EEXIST:
raise
else:
# Add the whole directory.
run('bzr', 'add', test_dir)
move_it(old_path, new_path)
dir_path, file_name = os.path.split(old_path)
if file_name.endswith('py') and not file_name.startswith('test_'):
update_helper_imports(old_path, new_path)
def update_helper_imports(old_path, new_path):
"""Fix the references to the test helper."""
old_dir_path, file_name = os.path.split(old_path)
old_module_path = file2module(old_dir_path).replace('.', '\\.')
module_name, dummy = os.path.splitext(file_name)
new_module_path = file2module(os.path.dirname(new_path))
source = r'\b%s(\.| import )%s\b' % (old_module_path, module_name)
target = r'%s\1%s' % (new_module_path, module_name)
root_dirs = ['cronscripts', 'lib/canonical', 'lib/lp']
file_pattern = '\.(py|txt|zcml)$'
print source, target
print " Updating references:"
for root_dir in root_dirs:
for summary in find_matches(
root_dir, file_pattern, source, substitution=target):
print " * %(file_path)s" % summary
def setup_test_harnesses(app_name):
"""Create the doctest harnesses."""
app_path = os.path.join(NEW_TOP, app_name)
doctest_path = os.path.join(app_path, 'doc')
doctests = [file_name
for file_name in os.listdir(doctest_path)
if file_name.endswith('.txt')]
print 'Installing doctest harnesses'
install_doctest_suite(
'test_doc.py', os.path.join(app_path, 'tests'), doctests=doctests)
install_doctest_suite(
'test_views.py', os.path.join(app_path, 'browser', 'tests'))
def install_doctest_suite(file_name, dir_path, doctests=None):
"""Copy the simple doctest builder."""
test_doc_path = os.path.join(
os.path.dirname(__file__), file_name)
test_doc_file = open(test_doc_path, 'r')
try:
test_doc = test_doc_file.read()
finally:
test_doc_file.close()
test_doc_path = os.path.join(dir_path, file_name)
if os.path.isfile(test_doc_path):
# This harness was made in a previous run.
print " Skipping %s, it was made in a previous run" % test_doc_path
return
test_doc_file = open(test_doc_path, 'w')
try:
test_doc_file.write(test_doc)
finally:
test_doc_file.close()
bzr_add([test_doc_path])
def handle_py_file(old_path, new_path, subdir):
"""Migrate python files."""
if subdir in APP_DIRECTORIES:
# We need the full path, including file name.
move_it(old_path, new_path)
return True
else:
return False
def get_all_module_members(app_name):
"""Return a dict of dicts of lists; package, module, members."""
all_members = {}
package_names = ['interfaces', 'model', 'browser', 'components']
member_pattern = r'^(?:class|def) (?P<name>[\w]*)'
for package_name in package_names:
root_dir = os.path.join(NEW_TOP, app_name, package_name)
module_names = {}
for summary in find_matches(root_dir, 'py$', member_pattern):
members = []
for line in summary['lines']:
members.append(line['match'].group('name'))
module_name, dummy = os.path.splitext(
os.path.basename(summary['file_path']))
# Reverse sorting avoids false-positive matches in REs.
module_names[module_name] = sorted(members, reverse=True)
all_members[package_name] = module_names
return all_members
def one_true_import(app_name, all_members):
"""Replace glob inports from interfaces to avoid circular imports."""
app_path = os.path.join(NEW_TOP, app_name)
print "Replace glob inports from interfaces to avoid circular imports."
all_interfaces = get_all_interfaces()
for file_path in find_files(app_path, file_pattern='py$'):
fix_file_true_import(file_path, all_interfaces)
def fix_file_true_import(file_path, all_interfaces):
"""Fix the interface imports in a file."""
from textwrap import fill
bad_pattern = 'from package.path.that.is.globbed import'
delimiters_pattern = re.compile(r'[,()]+')
import_lines = []
content = []
in_import = False
changed = False
file_ = open(file_path, 'r')
try:
for line in file_:
if in_import and len(line) > 1 and line[0] != ' ':
in_import = False
# Build a dict of interfaces used.
bad_import = delimiters_pattern.sub(
' ', ''.join(import_lines))
identifiers = bad_import.split()[3:]
modules = {}
for identifier in identifiers:
if identifier not in all_interfaces:
print ' * missing %s' % identifier
continue
modules.setdefault(
all_interfaces[identifier], []).append(identifier)
good_imports = []
# Build the import code from the dict.
for module_path in sorted(modules):
symbols = ', '.join(sorted(modules[module_path]))
if len(symbols) > 78 - len(bad_pattern):
symbols = '(\n%s)' % fill(
symbols, width=78,
initial_indent=' ', subsequent_indent=' ')
good_imports.append(
'from %s import %s\n' % (module_path, symbols))
# Insert the good imports into the module.
content.extend(good_imports)
if line.startswith(bad_pattern):
in_import = True
changed = True
import_lines = []
print ' Fixing interface imports in %s' % file_path
if in_import:
import_lines.append(line)
else:
content.append(line)
finally:
file_.close()
if changed:
file_ = open(file_path, 'w')
try:
file_.write(''.join(content))
finally:
file_.close()
def get_all_interfaces():
"""return a dict of interface member and module path."""
# {'IPersonSet', 'lp.registrty.interfaces.person'}
all_interfaces = {}
member_pattern = r'^(?:class |def )*(?P<name>[\w]*)'
for summary in find_matches(
'.', '(canonical|lp)/.*/interfaces.*\.py$', member_pattern):
module_path, dummy = os.path.splitext(summary['file_path'])
module_path = module_path.replace('./lib/', '')
assert module_path.startswith('lp') or module_path.startswith('ca'), (
'!! Bad module path.')
module_path = module_path.replace('/', '.')
for line in summary['lines']:
all_interfaces[line['match'].group('name')] = module_path
return all_interfaces
def handle_templates(app):
"""Migrate the page templates referenced in the zcml."""
new_browser_path = os.path.join(NEW_TOP, TLA_MAP[app], 'browser')
new_template_path = os.path.join(NEW_TOP, TLA_MAP[app], 'templates')
templates = set()
missing_templates = []
shared_templates = []
for summary in find_matches(
new_browser_path, '\.zcml$', r'template="\.\./([^"]+)"'):
for line in summary['lines']:
file_name = line['match'].group(1)
templates.add(os.path.join(OLD_TOP, file_name))
# Some views have the template file in the code.
for summary in find_matches(
new_browser_path, '\.py$', r"""\.\./(templates/[^"']+)"""):
for line in summary['lines']:
file_name = line['match'].group(1)
if 'xrds' in file_name:
# xrds files belong to OpenID and account. Fix the single
# reference in the registry tree.
old_template = (
"../../../canonical/launchpad/templates/person-xrds.pt")
for dummy in find_matches(
new_browser_path, 'person.py',
'../templates/person-xrds.pt', substitution=old_template):
pass
continue
templates.add(os.path.join(OLD_TOP, file_name))
print "Processing templates"
for template_path in templates:
if not os.path.isfile(template_path):
missing_templates.append(template_path)
continue
if is_shared_template(template_path):
shared_templates.append(template_path)
continue
bzr_move_file(template_path, new_template_path)
if len(missing_templates) > 0:
print "zcml references unknown templates:"
for file_path in missing_templates:
print ' %s' % file_path
if len(shared_templates) > 0:
print "Warning: many apps reference these templates (fix by hand):"
for template_path in shared_templates:
file_name = os.path.basename(template_path)
print ' %s' % file_name
# Update the template reference in the browser/*zcml.
pattern = r'(template=")\.\./(templates/%s)"' % file_name
substitution = r'\1../../../canonical/launchpad/\2"'
for summary in find_matches(
new_browser_path, '\.zcml$', pattern,
substitution=substitution):
pass
def is_shared_template(template_path):
"""Return true if the template is referenced in the old zcml."""
old_zcml_path = os.path.join(OLD_TOP, 'zcml')
file_name = os.path.basename(template_path)
for dummy in find_matches(old_zcml_path, '\.zcml$', file_name):
return True
return False
def main(ctl_data, apps, opts):
"""Migrate applications."""
# Get a dict keyed by app TLA with all files for that app.
app_to_files = convert_ctl_data(ctl_data)
if len(apps) == 1 and apps[0] == 'all':
apps = app_to_files.keys()
not_moved = []
for app in apps:
if app not in app_to_files:
print "No files tagged for app", app
continue
if app not in TLA_MAP:
print 'Unknown file owner:', app
continue
app_name = TLA_MAP[app]
make_tree(app)
if not opts.move:
continue
app_files = app_to_files[app]
for fpath in app_files:
if fpath.endswith('.zcml'):
# ZCML is processed after modules are moved.
not_moved.append(fpath)
continue
print "Processing:", fpath
full_path = os.path.join(OLD_TOP, fpath)
if not os.path.exists(full_path):
# The module has already been moved, ignore.
continue
to_path = map_filename(fpath)
path, file_name = os.path.split(to_path)
spew(" to_path = %s", to_path)
new_path_to_dir = os.path.join(NEW_TOP, app_name, path)
new_path_to_fn = os.path.join(NEW_TOP, app_name, to_path)
# Special cases.
if set(fpath.split(os.sep)) & TEST_PATHS:
handle_test(full_path, new_path_to_fn)
continue
subdir = to_path.split(os.sep)[0]
if subdir in ['scripts']:
handle_script(full_path, new_path_to_dir)
continue
# Only process python modules.
if file_name.endswith('.py'):
if handle_py_file(full_path, new_path_to_fn, subdir):
continue
not_moved.append(fpath)
# Create the test harnesses for the moved tests.
setup_test_harnesses(app_name)
# Replace glob imports from interfaces to avoid circular nonsense.
app_members = get_all_module_members(app_name)
one_true_import(app_name, app_members)
# Migrate the zcml.
handle_zcml(
app_name, OLD_TOP, NEW_TOP, app_files, app_members, not_moved)
# Move templates referenced by the moved zcml and view classes.
handle_templates(app)
# Warn about the files that weren't moved.
if len(not_moved) > 0:
print ("Warning: the following did not have a rule to "
"move them. They are left unchanged.")
for nm in not_moved:
print " ", nm
# Warn about collisions.
if len(COLLIDED) > 0:
print ("Warning: the following files collided and need to be "
"manually fixed.")
for pow in COLLIDED:
print " ", pow
if __name__ == '__main__':
parser, opts, args = parse_args()
if len(args) < 2:
parser.error('Control file and at least one app is required')
ctl_fn = args[0]
apps = args[1:]
ctl_data = open(ctl_fn, 'r').readlines()
main(ctl_data, apps, opts)
|