1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
|
# Copyright 2009 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
# pylint: disable-msg=E0213
"""The Launchpad code hosting file system.
The way Launchpad presents branches is very different from the way it stores
them. Externally, branches are reached using URLs that look like
<schema>://launchpad.net/~owner/product/branch-name. Internally, they are
stored by branch ID. Branch 1 is stored at 00/00/00/01 and branch 10 is stored
at 00/00/00/0A. Further, these branches might not be stored on the same
physical machine.
This means that our services need to translate the external paths into
internal paths.
We also want to let users create new branches on Launchpad simply by pushing
them up.
This means our services must detect events like 'make directory' and 'unlock
branch' and translate them into Launchpad operations like 'create branch' and
'request mirror' before performing those operations.
So, we have a `LaunchpadServer` which implements the core operations --
translate a path, make a branch and request a mirror -- in terms of virtual
paths.
This server does most of its work by querying an XML-RPC server that provides
the `IBranchFileSystem` interface and passing what that returns to a
`ITransportDispatch` object.
We hook the `LaunchpadServer` into Bazaar by implementing a
`AsyncVirtualTransport`, a `bzrlib.transport.Transport` that wraps all of its
operations so that they are translated by an object that implements
`translateVirtualPath`. See transport.py for more information.
This virtual transport isn't quite enough, since it only does dumb path
translation. We also need to be able to interpret filesystem events in terms
of Launchpad branches. To do this, we provide a `LaunchpadTransport` that
hooks into operations like `mkdir` and ask the `LaunchpadServer` to make a
branch if appropriate.
"""
__metaclass__ = type
__all__ = [
'AsyncLaunchpadTransport',
'BadUrl',
'BadUrlLaunchpad',
'BadUrlScheme',
'BadUrlSsh',
'branch_id_to_path',
'BranchPolicy',
'DirectDatabaseLaunchpadServer',
'get_lp_server',
'get_ro_server',
'get_rw_server',
'make_branch_mirrorer',
'LaunchpadInternalServer',
'LaunchpadServer',
]
import xmlrpclib
from bzrlib.branch import Branch
from bzrlib.bzrdir import BzrDir, BzrDirFormat
from bzrlib.config import TransportConfig
from bzrlib.errors import NoSuchFile, PermissionDenied, TransportNotPossible
from bzrlib.plugins.loom.branch import LoomSupport
from bzrlib.smart.request import jail_info
from bzrlib.transport import get_transport
from bzrlib.transport.memory import MemoryServer
from bzrlib import urlutils
from lazr.uri import URI
from twisted.internet import defer
from twisted.python import failure, log
from zope.component import getUtility
from zope.interface import implements, Interface
from lp.codehosting.bzrutils import get_stacked_on_url
from lp.codehosting.vfs.branchfsclient import (
BlockingProxy, BranchFileSystemClient)
from lp.codehosting.vfs.transport import (
AsyncVirtualServer, AsyncVirtualTransport, TranslationError,
get_chrooted_transport, get_readonly_transport)
from canonical.config import config
from canonical.launchpad.xmlrpc import faults
from lp.code.enums import BranchType
from lp.code.interfaces.branchlookup import IBranchLookup
from lp.code.interfaces.codehosting import (
BRANCH_TRANSPORT, CONTROL_TRANSPORT, LAUNCHPAD_SERVICES)
from lp.services.twistedsupport.xmlrpc import trap_fault
class BadUrl(Exception):
"""Tried to access a branch from a bad URL."""
class BadUrlSsh(BadUrl):
"""Tried to access a branch from sftp or bzr+ssh."""
class BadUrlLaunchpad(BadUrl):
"""Tried to access a branch from launchpad.net."""
class BadUrlScheme(BadUrl):
"""Found a URL with an untrusted scheme."""
def __init__(self, scheme, url):
BadUrl.__init__(self, scheme, url)
self.scheme = scheme
# The directories allowed directly beneath a branch directory. These are the
# directories that Bazaar creates as part of regular operation. We support
# only two numbered backups to avoid indefinite space usage.
ALLOWED_DIRECTORIES = ('.bzr', '.bzr.backup', 'backup.bzr', 'backup.bzr.~1~',
'backup.bzr.~2~')
FORBIDDEN_DIRECTORY_ERROR = (
"Cannot create '%s'. Only Bazaar branches are allowed.")
class NotABranchPath(TranslationError):
"""Raised when we cannot translate a virtual URL fragment to a branch.
In particular, this is raised when there is some intrinsic deficiency in
the path itself.
"""
_fmt = ("Could not translate %(virtual_url_fragment)r to branch. "
"%(reason)s")
class UnknownTransportType(Exception):
"""Raised when we don't know the transport type."""
def branch_id_to_path(branch_id):
"""Convert the given branch ID into NN/NN/NN/NN form, where NN is a two
digit hexadecimal number.
Some filesystems are not capable of dealing with large numbers of inodes.
The codehosting system has tens of thousands of branches and thus splits
the branches into several directories. The Launchpad id is used in order
to determine the splitting.
"""
h = "%08x" % int(branch_id)
return '%s/%s/%s/%s' % (h[:2], h[2:4], h[4:6], h[6:])
def get_path_segments(path, maximum_segments=-1):
"""Break up the given path into segments.
If 'path' ends with a trailing slash, then the final empty segment is
ignored.
"""
return path.strip('/').split('/', maximum_segments)
def is_lock_directory(absolute_path):
"""Is 'absolute_path' a Bazaar branch lock directory?"""
return absolute_path.endswith('/.bzr/branch/lock/held')
def get_ro_server():
"""Get a Launchpad internal server for scanning branches."""
proxy = xmlrpclib.ServerProxy(config.codehosting.codehosting_endpoint)
codehosting_endpoint = BlockingProxy(proxy)
branch_transport = get_readonly_transport(
get_transport(config.codehosting.internal_branch_by_id_root))
return LaunchpadInternalServer(
'lp-internal:///', codehosting_endpoint, branch_transport)
def get_rw_server(direct_database=False):
"""Get a server that can write to the Launchpad branch vfs.
You can only call this usefully on the codehost -- the transport this
server provides are backed onto file:/// URLs.
:param direct_database: if True, use a server implementation that talks
directly to the database. If False, the default, use a server
implementation that talks to the internal XML-RPC server.
"""
transport = get_chrooted_transport(
config.codehosting.mirrored_branches_root, mkdir=True)
if direct_database:
return DirectDatabaseLaunchpadServer('lp-internal:///', transport)
else:
proxy = xmlrpclib.ServerProxy(config.codehosting.codehosting_endpoint)
codehosting_endpoint = BlockingProxy(proxy)
return LaunchpadInternalServer(
'lp-internal:///', codehosting_endpoint, transport)
class ITransportDispatch(Interface):
"""Turns descriptions of transports into transports."""
def makeTransport(transport_tuple):
"""Return a transport and trailing path for 'transport_tuple'.
:param transport_tuple: a tuple of (transport_type, transport_data,
trailing_path), as returned by IBranchFileSystem['translatePath'].
:return: A transport and a path on that transport that point to a
place that matches the one described in transport_tuple.
:rtype: (`bzrlib.transport.Transport`, str)
"""
class BranchTransportDispatch:
"""Turns BRANCH_TRANSPORT tuples into transports that point to branches.
This transport dispatch knows how branches are laid out on the disk in a
particular "area". It doesn't know anything about the "hosted" or
"mirrored" areas.
This is used directly by our internal services (puller and scanner).
"""
implements(ITransportDispatch)
def __init__(self, base_transport):
self.base_transport = base_transport
def _checkPath(self, path_on_branch):
"""Raise an error if `path_on_branch` is not valid.
This allows us to enforce a certain level of policy about what goes
into a branch directory on Launchpad. Specifically, we do not allow
arbitrary files at the top-level, we only allow Bazaar control
directories, and backups of same.
:raise PermissionDenied: if `path_on_branch` is forbidden.
"""
if path_on_branch == '':
return
segments = get_path_segments(path_on_branch)
if segments[0] not in ALLOWED_DIRECTORIES:
raise PermissionDenied(
FORBIDDEN_DIRECTORY_ERROR % (segments[0],))
def makeTransport(self, transport_tuple):
"""See `ITransportDispatch`.
:raise PermissionDenied: If the path on the branch's transport is
forbidden because it's not in ALLOWED_DIRECTORIES.
"""
transport_type, data, trailing_path = transport_tuple
if transport_type != BRANCH_TRANSPORT:
raise UnknownTransportType(transport_type)
self._checkPath(trailing_path)
transport = self.base_transport.clone(branch_id_to_path(data['id']))
try:
transport.create_prefix()
except TransportNotPossible:
# Silently ignore TransportNotPossible. This is raised when the
# base transport is read-only.
pass
return transport, trailing_path
class TransportDispatch:
"""Make transports for hosted branch areas and virtual control dirs.
This transport dispatch knows about whether a particular branch should be
served read-write or read-only. It also knows how to serve .bzr control
directories for products (to enable default stacking).
This is used for the rich codehosting VFS that we serve publically.
"""
implements(ITransportDispatch)
def __init__(self, rw_transport):
self._rw_dispatch = BranchTransportDispatch(rw_transport)
self._ro_dispatch = BranchTransportDispatch(
get_readonly_transport(rw_transport))
self._transport_factories = {
BRANCH_TRANSPORT: self._makeBranchTransport,
CONTROL_TRANSPORT: self._makeControlTransport,
}
def makeTransport(self, transport_tuple):
transport_type, data, trailing_path = transport_tuple
factory = self._transport_factories[transport_type]
data['trailing_path'] = trailing_path
return factory(**data), trailing_path
def _makeBranchTransport(self, id, writable, trailing_path=''):
if writable:
dispatch = self._rw_dispatch
else:
dispatch = self._ro_dispatch
transport, ignored = dispatch.makeTransport(
(BRANCH_TRANSPORT, dict(id=id), trailing_path))
return transport
def _makeControlTransport(self, default_stack_on, trailing_path=None):
"""Make a transport that points to a control directory.
A control directory is a .bzr directory containing a 'control.conf'
file. This is used to specify configuration for branches created
underneath the directory that contains the control directory.
:param default_stack_on: The default stacked-on branch URL for
branches that respect this control directory. If empty, then
we'll return an empty memory transport.
:return: A read-only `MemoryTransport` containing a working BzrDir,
configured to use the given default stacked-on location.
"""
memory_server = MemoryServer()
memory_server.start_server()
transport = get_transport(memory_server.get_url())
if default_stack_on == '':
return transport
format = BzrDirFormat.get_default_format()
bzrdir = format.initialize_on_transport(transport)
bzrdir.get_config().set_default_stack_on(
urlutils.unescape(default_stack_on))
return get_readonly_transport(transport)
class _BaseLaunchpadServer(AsyncVirtualServer):
"""Bazaar `Server` for translating Lanuchpad paths via XML-RPC.
This server provides facilities for transports that use a virtual
filesystem, backed by an XML-RPC server.
For more information, see the module docstring.
:ivar _branchfs_client: An object that has a method 'translatePath' that
returns a Deferred that fires information about how a path can be
translated into a transport. See `IBranchFilesystem['translatePath']`.
:ivar _transport_dispatch: An `ITransportDispatch` provider used to
convert the data from the branchfs client into an actual transport and
path on that transport.
"""
def __init__(self, scheme, codehosting_api, user_id,
seen_new_branch_hook=None):
"""Construct a LaunchpadServer.
:param scheme: The URL scheme to use.
:param codehosting_api: An XML-RPC client that implements callRemote.
:param user_id: The database ID for the user who is accessing
branches.
:param seen_new_branch_hook: A callable that will be called once for
each branch accessed via this server.
"""
AsyncVirtualServer.__init__(self, scheme)
self._branchfs_client = BranchFileSystemClient(
codehosting_api, user_id,
seen_new_branch_hook=seen_new_branch_hook)
self._is_start_server = False
def translateVirtualPath(self, virtual_url_fragment):
"""See `AsyncVirtualServer.translateVirtualPath`.
Call 'translatePath' on the branchfs client with the fragment and then
use 'makeTransport' on the _transport_dispatch to translate that
result into a transport and trailing path.
"""
deferred = self._branchfs_client.translatePath(
'/' + virtual_url_fragment)
def path_not_translated(failure):
trap_fault(
failure, faults.PathTranslationError, faults.PermissionDenied)
raise NoSuchFile(virtual_url_fragment)
def unknown_transport_type(failure):
failure.trap(UnknownTransportType)
raise NoSuchFile(virtual_url_fragment)
deferred.addCallbacks(
self._transport_dispatch.makeTransport, path_not_translated)
deferred.addErrback(unknown_transport_type)
return deferred
class LaunchpadInternalServer(_BaseLaunchpadServer):
"""Server for Launchpad internal services.
This server provides access to a transport using the Launchpad virtual
filesystem. Unlike the `LaunchpadServer`, it backs onto a single transport
and doesn't do any permissions work.
Intended for use with the branch puller and scanner.
"""
def __init__(self, scheme, codehosting_api, branch_transport):
"""Construct a `LaunchpadInternalServer`.
:param scheme: The URL scheme to use.
:param codehosting_api: An object that provides a 'translatePath'
method.
:param branch_transport: A Bazaar `Transport` that refers to an
area where Launchpad branches are stored, generally either the
hosted or mirrored areas.
"""
super(LaunchpadInternalServer, self).__init__(
scheme, codehosting_api, LAUNCHPAD_SERVICES)
self._transport_dispatch = BranchTransportDispatch(branch_transport)
def start_server(self):
super(LaunchpadInternalServer, self).start_server()
try:
self._transport_dispatch.base_transport.ensure_base()
except TransportNotPossible:
pass
def destroy(self):
"""Delete the on-disk branches and tear down."""
self._transport_dispatch.base_transport.delete_tree('.')
self.stop_server()
class DirectDatabaseLaunchpadServer(AsyncVirtualServer):
def __init__(self, scheme, branch_transport):
AsyncVirtualServer.__init__(self, scheme)
self._transport_dispatch = BranchTransportDispatch(branch_transport)
def start_server(self):
super(DirectDatabaseLaunchpadServer, self).start_server()
try:
self._transport_dispatch.base_transport.ensure_base()
except TransportNotPossible:
pass
def destroy(self):
"""Delete the on-disk branches and tear down."""
self._transport_dispatch.base_transport.delete_tree('.')
self.stop_server()
def translateVirtualPath(self, virtual_url_fragment):
"""See `AsyncVirtualServer.translateVirtualPath`.
This implementation connects to the database directly.
"""
deferred = defer.succeed(
getUtility(IBranchLookup).getIdAndTrailingPath(
virtual_url_fragment))
def process_result((branch_id, trailing)):
if branch_id is None:
raise NoSuchFile(virtual_url_fragment)
else:
return self._transport_dispatch.makeTransport(
(BRANCH_TRANSPORT, dict(id=branch_id), trailing[1:]))
deferred.addCallback(process_result)
return deferred
class AsyncLaunchpadTransport(AsyncVirtualTransport):
"""Virtual transport to implement the Launchpad VFS for branches.
This implements a few hooks to translate filesystem operations (such as
making a certain kind of directory) into Launchpad operations (such as
creating a branch in the database).
It also converts the Launchpad-specific translation errors (such as 'not a
valid branch path') into Bazaar errors (such as 'no such file').
"""
def mkdir(self, relpath, mode=None):
# We hook into mkdir so that we can request the creation of a branch
# and so that we can provide useful errors in the special case where
# the user tries to make a directory like "~foo/bar". That is, a
# directory that has too little information to be translated into a
# Launchpad branch.
deferred = AsyncVirtualTransport._getUnderylingTransportAndPath(
self, relpath)
def maybe_make_branch_in_db(failure):
# Looks like we are trying to make a branch.
failure.trap(NoSuchFile)
return self.server.createBranch(self._abspath(relpath))
def real_mkdir((transport, path)):
return getattr(transport, 'mkdir')(path, mode)
deferred.addCallback(real_mkdir)
deferred.addErrback(maybe_make_branch_in_db)
return deferred
def rename(self, rel_from, rel_to):
# We hook into rename to catch the "unlock branch" event, so that we
# can request a mirror once a branch is unlocked.
abs_from = self._abspath(rel_from)
if is_lock_directory(abs_from):
deferred = self.server.branchChanged(abs_from)
else:
deferred = defer.succeed(None)
deferred = deferred.addCallback(
lambda ignored: AsyncVirtualTransport.rename(
self, rel_from, rel_to))
return deferred
def rmdir(self, relpath):
# We hook into rmdir in order to prevent users from deleting branches,
# products and people from the VFS.
virtual_url_fragment = self._abspath(relpath)
path_segments = virtual_url_fragment.lstrip('/').split('/')
# XXX: JonathanLange 2008-11-19 bug=300551: This code assumes stuff
# about the VFS! We need to figure out the best way to delegate the
# decision about permission-to-delete to the XML-RPC server.
if len(path_segments) <= 3:
return defer.fail(
failure.Failure(PermissionDenied(virtual_url_fragment)))
return AsyncVirtualTransport.rmdir(self, relpath)
class LaunchpadServer(_BaseLaunchpadServer):
"""The Server used for the public SSH codehosting service.
This server provides a VFS that backs onto a transport that stores
branches by id. The TransportDispatch object takes care of showing a
writeable or read-only view of each branch as appropriate.
In addition to basic VFS operations, this server provides operations for
creating a branch and requesting for a branch to be mirrored. The
associated transport, `AsyncLaunchpadTransport`, has hooks in certain
filesystem-level operations to trigger these.
"""
asyncTransportFactory = AsyncLaunchpadTransport
def __init__(self, codehosting_api, user_id, branch_transport,
seen_new_branch_hook=None):
"""Construct a `LaunchpadServer`.
See `_BaseLaunchpadServer` for more information.
:param codehosting_api: An object that has 'createBranch' and
'branchChanged' methods in addition to a 'translatePath' method.
These methods should return Deferreds.
XXX: JonathanLange 2008-11-19: Specify this interface better.
:param user_id: The database ID of the user to connect as.
:param branch_transport: A Bazaar `Transport` that points to the
"hosted" area of Launchpad. See module docstring for more
information.
:param seen_new_branch_hook: A callable that will be called once for
each branch accessed via this server.
"""
scheme = 'lp-%d:///' % id(self)
super(LaunchpadServer, self).__init__(
scheme, codehosting_api, user_id, seen_new_branch_hook)
self._transport_dispatch = TransportDispatch(branch_transport)
def createBranch(self, virtual_url_fragment):
"""Make a new directory for the given virtual URL fragment.
If `virtual_url_fragment` is a branch directory, create the branch in
the database, then create a matching directory on the backing
transport.
:param virtual_url_fragment: A virtual path to be translated.
:raise NotABranchPath: If `virtual_path` does not have at least a
valid path to a branch.
:raise NotEnoughInformation: If `virtual_path` does not map to a
branch.
:raise PermissionDenied: If the branch cannot be created in the
database. This might indicate that the branch already exists, or
that its creation is forbidden by a policy.
:raise Fault: If the XML-RPC server raises errors.
"""
deferred = self._branchfs_client.createBranch(virtual_url_fragment)
def translate_fault(failure):
# We turn faults.NotFound into a PermissionDenied, even
# though one might think that it would make sense to raise
# NoSuchFile. Sadly, raising that makes the client do "clever"
# things like say "Parent directory of
# bzr+ssh://bazaar.launchpad.dev/~noone/firefox/branch does not
# exist. You may supply --create-prefix to create all leading
# parent directories", which is just misleading.
fault = trap_fault(
failure, faults.NotFound, faults.PermissionDenied)
faultString = fault.faultString
if isinstance(faultString, unicode):
faultString = faultString.encode('utf-8')
raise PermissionDenied(virtual_url_fragment, faultString)
return deferred.addErrback(translate_fault)
def _normalize_stacked_on_url(self, branch):
"""Normalize and return the stacked-on location of `branch`.
In the common case, `branch` will either be unstacked or stacked on a
relative path, in which case this is very easy: just return the
location.
If `branch` is stacked on the absolute URL of another Launchpad
branch, we normalize this to a relative path (mutating the branch) and
return the relative path.
If `branch` is stacked on some other absolute URL we don't recognise,
we just return that and rely on the `branchChanged` XML-RPC method
recording a complaint in the appropriate place.
"""
stacked_on_url = get_stacked_on_url(branch)
if stacked_on_url is None:
return None
if '://' not in stacked_on_url:
# Assume it's a relative path.
return stacked_on_url
uri = URI(stacked_on_url)
if uri.scheme not in ['http', 'bzr+ssh', 'sftp']:
return stacked_on_url
launchpad_domain = config.vhost.mainsite.hostname
if not uri.underDomain(launchpad_domain):
return stacked_on_url
# We use TransportConfig directly because the branch
# is still locked at this point! We're effectively
# 'borrowing' the lock that is being released.
branch_config = TransportConfig(branch._transport, 'branch.conf')
branch_config.set_option(uri.path, 'stacked_on_location')
return uri.path
def branchChanged(self, virtual_url_fragment):
"""Notify Launchpad of a change to the a branch.
This method tries hard to not exit via an exception, because the
client side experience is not good in that case. Instead, log.err()
is called, which will result in an OOPS being logged.
:param virtual_url_fragment: A url fragment that points to a path
owned by a branch.
"""
deferred = self._branchfs_client.translatePath(
'/' + virtual_url_fragment)
def got_path_info((transport_type, data, trailing_path)):
if transport_type != BRANCH_TRANSPORT:
raise NotABranchPath(virtual_url_fragment)
transport, _ = self._transport_dispatch.makeTransport(
(transport_type, data, trailing_path))
if jail_info.transports:
jail_info.transports.append(transport)
try:
branch = BzrDir.open_from_transport(transport).open_branch(
ignore_fallbacks=True)
last_revision = branch.last_revision()
stacked_on_url = self._normalize_stacked_on_url(branch)
# XXX: Aaron Bentley 2008-06-13
# Bazaar does not provide a public API for learning about
# format markers. Fix this in Bazaar, then here.
control_string = branch.bzrdir._format.get_format_string()
branch_string = branch._format.get_format_string()
repository_string = \
branch.repository._format.get_format_string()
finally:
if jail_info.transports:
jail_info.transports.remove(transport)
if stacked_on_url is None:
stacked_on_url = ''
return self._branchfs_client.branchChanged(
data['id'], stacked_on_url, last_revision,
control_string, branch_string, repository_string)
# It gets really confusing if we raise an exception from this method
# (the branch remains locked, but this isn't obvious to the client) so
# just log the error, which will result in an OOPS being logged.
return deferred.addCallback(got_path_info).addErrback(log.err)
def get_lp_server(user_id, codehosting_endpoint_url=None, branch_url=None,
seen_new_branch_hook=None, branch_transport=None):
"""Create a Launchpad server.
:param user_id: A unique database ID of the user whose branches are
being served.
:param codehosting_endpoint_url: URL for the branch file system end-point.
:param hosted_directory: Where the branches are uploaded to.
:param mirror_directory: Where all Launchpad branches are mirrored.
:param seen_new_branch_hook:
:return: A `LaunchpadServer`.
"""
# Get the defaults from the config.
if codehosting_endpoint_url is None:
codehosting_endpoint_url = config.codehosting.codehosting_endpoint
if branch_url is None:
if branch_transport is None:
branch_url = config.codehosting.mirrored_branches_root
branch_transport = get_chrooted_transport(branch_url)
else:
if branch_transport is None:
branch_transport = get_chrooted_transport(branch_url)
else:
raise AssertionError(
"can't supply both branch_url and branch_transport!")
codehosting_client = xmlrpclib.ServerProxy(codehosting_endpoint_url)
lp_server = LaunchpadServer(
BlockingProxy(codehosting_client), user_id, branch_transport,
seen_new_branch_hook)
return lp_server
class BranchPolicy:
"""Policy on how to mirror branches.
In particular, a policy determines which branches are safe to mirror by
checking their URLs and deciding whether or not to follow branch
references. A policy also determines how the mirrors of branches should be
stacked.
"""
def createDestinationBranch(self, source_branch, destination_url):
"""Create a destination branch for 'source_branch'.
Creates a branch at 'destination_url' that is has the same format as
'source_branch'. Any content already at 'destination_url' will be
deleted. Generally the new branch will have no revisions, but they
will be copied for import branches, because this can be done safely
and efficiently with a vfs-level copy (see `ImportedBranchPolicy`,
below).
:param source_branch: The Bazaar branch that will be mirrored.
:param destination_url: The place to make the destination branch. This
URL must point to a writable location.
:return: The destination branch.
"""
dest_transport = get_transport(destination_url)
if dest_transport.has('.'):
dest_transport.delete_tree('.')
if isinstance(source_branch, LoomSupport):
# Looms suck.
revision_id = None
else:
revision_id = 'null:'
source_branch.bzrdir.clone_on_transport(
dest_transport, revision_id=revision_id)
return Branch.open(destination_url)
def getStackedOnURLForDestinationBranch(self, source_branch,
destination_url):
"""Get the stacked on URL for `source_branch`.
In particular, the URL it should be stacked on when it is mirrored to
`destination_url`.
"""
return None
def shouldFollowReferences(self):
"""Whether we traverse references when mirroring.
Subclasses must override this method.
If we encounter a branch reference and this returns false, an error is
raised.
:returns: A boolean to indicate whether to follow a branch reference.
"""
raise NotImplementedError(self.shouldFollowReferences)
def transformFallbackLocation(self, branch, url):
"""Validate, maybe modify, 'url' to be used as a stacked-on location.
:param branch: The branch that is being opened.
:param url: The URL that the branch provides for its stacked-on
location.
:return: (new_url, check) where 'new_url' is the URL of the branch to
actually open and 'check' is true if 'new_url' needs to be
validated by checkAndFollowBranchReference.
"""
raise NotImplementedError(self.transformFallbackLocation)
def checkOneURL(self, url):
"""Check the safety of the source URL.
Subclasses must override this method.
:param url: The source URL to check.
:raise BadUrl: subclasses are expected to raise this or a subclass
when it finds a URL it deems to be unsafe.
"""
raise NotImplementedError(self.checkOneURL)
class MirroredBranchPolicy(BranchPolicy):
"""Mirroring policy for MIRRORED branches.
In summary:
- follow references,
- only open non-Launchpad http: and https: URLs.
"""
def __init__(self, stacked_on_url=None):
self.stacked_on_url = stacked_on_url
def getStackedOnURLForDestinationBranch(self, source_branch,
destination_url):
"""See `BranchPolicy.getStackedOnURLForDestinationBranch`.
Mirrored branches are stacked on the default stacked-on branch of
their product, except when we're mirroring the default stacked-on
branch itself.
"""
if self.stacked_on_url is None:
return None
stacked_on_url = urlutils.join(destination_url, self.stacked_on_url)
if destination_url == stacked_on_url:
return None
return self.stacked_on_url
def shouldFollowReferences(self):
"""See `BranchPolicy.shouldFollowReferences`.
We traverse branch references for MIRRORED branches because they
provide a useful redirection mechanism and we want to be consistent
with the bzr command line.
"""
return True
def transformFallbackLocation(self, branch, url):
"""See `BranchPolicy.transformFallbackLocation`.
For mirrored branches, we stack on whatever the remote branch claims
to stack on, but this URL still needs to be checked.
"""
return urlutils.join(branch.base, url), True
def checkOneURL(self, url):
"""See `BranchPolicy.checkOneURL`.
We refuse to mirror from Launchpad or a ssh-like or file URL.
"""
# Avoid circular import
from lp.code.interfaces.branch import get_blacklisted_hostnames
uri = URI(url)
launchpad_domain = config.vhost.mainsite.hostname
if uri.underDomain(launchpad_domain):
raise BadUrlLaunchpad(url)
for hostname in get_blacklisted_hostnames():
if uri.underDomain(hostname):
raise BadUrl(url)
if uri.scheme in ['sftp', 'bzr+ssh']:
raise BadUrlSsh(url)
elif uri.scheme not in ['http', 'https']:
raise BadUrlScheme(uri.scheme, url)
class ImportedBranchPolicy(BranchPolicy):
"""Mirroring policy for IMPORTED branches.
In summary:
- don't follow references,
- assert the URLs start with the prefix we expect for imported branches.
"""
def createDestinationBranch(self, source_branch, destination_url):
"""See `BranchPolicy.createDestinationBranch`.
Because we control the process that creates import branches, a
vfs-level copy is safe and more efficient than a bzr fetch.
"""
source_transport = source_branch.bzrdir.root_transport
dest_transport = get_transport(destination_url)
while True:
# We loop until the remote file list before and after the copy is
# the same to catch the case where the remote side is being
# mutated as we copy it.
if dest_transport.has('.'):
dest_transport.delete_tree('.')
files_before = set(source_transport.iter_files_recursive())
source_transport.copy_tree_to_transport(dest_transport)
files_after = set(source_transport.iter_files_recursive())
if files_before == files_after:
break
return Branch.open_from_transport(dest_transport)
def shouldFollowReferences(self):
"""See `BranchPolicy.shouldFollowReferences`.
We do not traverse references for IMPORTED branches because the
code-import system should never produce branch references.
"""
return False
def transformFallbackLocation(self, branch, url):
"""See `BranchPolicy.transformFallbackLocation`.
Import branches should not be stacked, ever.
"""
raise AssertionError("Import branch unexpectedly stacked!")
def checkOneURL(self, url):
"""See `BranchPolicy.checkOneURL`.
If the URL we are mirroring from does not start how we expect the pull
URLs of import branches to start, something has gone badly wrong, so
we raise AssertionError if that's happened.
"""
if not url.startswith(config.launchpad.bzr_imports_root_url):
raise AssertionError(
"Bogus URL for imported branch: %r" % url)
def make_branch_mirrorer(branch_type, protocol=None,
mirror_stacked_on_url=None):
"""Create a `BranchMirrorer` with the appropriate `BranchPolicy`.
:param branch_type: A `BranchType` to select a policy by.
:param protocol: Optional protocol for the mirrorer to work with.
If given, its log will also be used.
:param mirror_stacked_on_url: For mirrored branches, the default URL
to stack on. Ignored for other branch types.
:return: A `BranchMirrorer`.
"""
# Avoid circular import
from lp.codehosting.puller.worker import BranchMirrorer
if branch_type == BranchType.MIRRORED:
policy = MirroredBranchPolicy(mirror_stacked_on_url)
elif branch_type == BranchType.IMPORTED:
policy = ImportedBranchPolicy()
else:
raise AssertionError(
"Unexpected branch type: %r" % branch_type)
if protocol is not None:
log_function = protocol.log
else:
log_function = None
return BranchMirrorer(policy, protocol, log_function)
|