« back to all changes in this revision
Viewing changes to lib/canonical/lazr/config.py
- browse files at revision 7482
- compare with another revision
- download diff
- download tarball
- view history from revision 7482
- Committer: Launchpad Patch Queue Manager
- Date: 2008-12-20 04:58:58 UTC
- mfrom: (7465.5.6 lazr-config-integration)
- Revision ID: launchpad@pqm.canonical.com-20081220045858-nctxabte1151hxgz
[r=barry] Integrate lazr.config and lazr.delegates into code
- files added:
- lib/lazr
- lib/lazr/config
- lib/lazr/delegates
- files removed:
- lib/canonical/lazr/config.py
- files modified:
- buildmailman.py
added
removed
lib/canonical/lazr/config.py
1
# Copyright 2004-2007 Canonical Ltd. All rights reserved.
2
3
"""Implementation classes for config."""
4
5
__metaclass__ = type
6
7
__all__ = [
8
'Config',
9
'ConfigData',
10
'ConfigSchema',
11
'ImplicitTypeSchema',
12
'ImplicitTypeSection',
13
'Section',
14
'SectionSchema',
15
'as_host_port',
16
'as_timedelta',
17
'as_username_groupname',
18
]
19
20
21
import StringIO
22
import datetime
23
import grp
24
import os
25
import pwd
26
import re
27
28
from ConfigParser import NoSectionError, RawConfigParser
29
from os.path import abspath, basename, dirname
30
from textwrap import dedent
31
32
from zope.interface import implements
33
34
from canonical.lazr.interfaces import (
35
ConfigErrors, ICategory, IConfigData, IConfigLoader, IConfigSchema,
36
InvalidSectionNameError, ISection, ISectionSchema, IStackableConfig,
37
NoCategoryError, NoConfigError, RedefinedSectionError, UnknownKeyError,
38
UnknownSectionError)
39
from canonical.lazr.decorates import decorates
40
41
42
def read_content(filename):
43
"""Return the content of a file at filename as a string."""
44
source_file = open(filename, 'r')
45
try:
46
raw_data = source_file.read()
47
finally:
48
source_file.close()
49
return raw_data
50
51
52
class SectionSchema:
53
"""See `ISectionSchema`."""
54
implements(ISectionSchema)
55
56
def __init__(self, name, options, is_optional=False):
57
"""Create an `ISectionSchema` from the name and options.
58
59
:param name: A string. The name of the ISectionSchema.
60
:param options: A dict of the key-value pairs in the ISectionSchema.
61
:param is_optional: A boolean. Is this section schema optional?
62
:raise `RedefinedKeyError`: if a keys is redefined in SectionSchema.
63
"""
64
# This method should raise RedefinedKeyError if the schema file
65
# redefines a key, but SafeConfigParser swallows redefined keys.
66
self.name = name
67
self._options = options
68
self.optional = is_optional
69
70
def __iter__(self):
71
"""See `ISectionSchema`"""
72
return self._options.iterkeys()
73
74
def __contains__(self, name):
75
"""See `ISectionSchema`"""
76
return name in self._options
77
78
def __getitem__(self, key):
79
"""See `ISectionSchema`"""
80
return self._options[key]
81
82
@property
83
def category_and_section_names(self):
84
"""See `ISectionSchema`."""
85
if '.' in self.name:
86
return tuple(self.name.split('.'))
87
else:
88
return (None, self.name)
89
90
91
class Section:
92
"""See `ISection`."""
93
implements(ISection)
94
decorates(ISectionSchema, context='schema')
95
96
def __init__(self, schema, _options=None):
97
"""Create an `ISection` from schema.
98
99
:param schema: The ISectionSchema that defines this ISection.
100
"""
101
# Use __dict__ because __getattr__ limits access to self.options.
102
self.__dict__['schema'] = schema
103
if _options is None:
104
_options = dict([(key, schema[key]) for key in schema])
105
self.__dict__['_options'] = _options
106
107
def __getitem__(self, key):
108
"""See `ISection`"""
109
return self._options[key]
110
111
def __getattr__(self, name):
112
"""See `ISection`."""
113
if name in self._options:
114
return self._options[name]
115
else:
116
raise AttributeError(
117
"No section key named %s." % name)
118
119
def __setattr__(self, name, value):
120
"""Callsites cannot mutate the config by direct manipulation."""
121
raise AttributeError("Config options cannot be set directly.")
122
123
@property
124
def category_and_section_names(self):
125
"""See `ISection`."""
126
return self.schema.category_and_section_names
127
128
def update(self, items):
129
"""Update the keys with new values.
130
131
:return: A list of `UnknownKeyError`s if the section does not have
132
the key. An empty list is returned if there are no errors.
133
"""
134
errors = []
135
for key, value in items:
136
if key in self._options:
137
self._options[key] = value
138
else:
139
msg = "%s does not have a %s key." % (self.name, key)
140
errors.append(UnknownKeyError(msg))
141
return errors
142
143
def clone(self):
144
"""Return a copy of this section.
145
146
The extension mechanism requires a copy of a section to prevent
147
mutation.
148
"""
149
new_section = self.__class__(self.schema, self._options.copy())
150
# XXX 2008-06-10 jamesh bug=237827:
151
# Evil legacy code sometimes assigns directly to the config
152
# section objects. Copy those attributes over.
153
new_section.__dict__.update(
154
dict((key, value) for (key, value) in self.__dict__.iteritems()
155
if key not in ['schema', '_options']))
156
return new_section
157
158
159
class ImplicitTypeSection(Section):
160
"""See `ISection`.
161
162
ImplicitTypeSection supports implicit conversion of key values to
163
simple datatypes. It accepts the same section data as Section; the
164
datatype information is not embedded in the schema or the config file.
165
"""
166
re_types = re.compile(r'''
167
(?P<false> ^false$) |
168
(?P<true> ^true$) |
169
(?P<none> ^none$) |
170
(?P<int> ^[+-]?\d+$) |
171
(?P<str> ^.*)
172
''', re.IGNORECASE | re.VERBOSE)
173
174
def _convert(self, value):
175
"""Return the value as the datatype the str appears to be.
176
177
Conversion rules:
178
* bool: a single word, 'true' or 'false', case insensitive.
179
* int: a single word that is a number. Signed is supported,
180
hex and octal numbers are not.
181
* str: anything else.
182
"""
183
match = self.re_types.match(value)
184
if match.group('false'):
185
return False
186
elif match.group('true'):
187
return True
188
elif match.group('none'):
189
return None
190
elif match.group('int'):
191
return int(value)
192
else:
193
# match.group('str'); just return the sripped value.
194
return value.strip()
195
196
def __getitem__(self, key):
197
"""See `ISection`."""
198
value = super(ImplicitTypeSection, self).__getitem__(key)
199
return self._convert(value)
200
201
def __getattr__(self, name):
202
"""See `ISection`."""
203
value = super(ImplicitTypeSection, self).__getattr__(name)
204
return self._convert(value)
205
206
207
class ConfigSchema:
208
"""See `IConfigSchema`."""
209
implements(IConfigSchema, IConfigLoader)
210
211
_section_factory = Section
212
213
def __init__(self, filename):
214
"""Load a configuration schema from the provided filename.
215
216
:raise `UnicodeDecodeError`: if the string contains non-ascii
217
characters.
218
:raise `RedefinedSectionError`: if a SectionSchema name is redefined.
219
:raise `InvalidSectionNameError`: if a SectionSchema name is
220
ill-formed.
221
"""
222
# XXX sinzui 2007-12-13:
223
# RawConfigParser permits redefinition and non-ascii characters.
224
# The raw schema data is examined before creating a config.
225
self.filename = filename
226
self.name = basename(filename)
227
self._section_schemas = {}
228
self._category_names = []
229
raw_schema = self._getRawSchema(filename)
230
parser = RawConfigParser()
231
parser.readfp(raw_schema, filename)
232
self._setSectionSchemasAndCategoryNames(parser)
233
234
235
def _getRawSchema(self, filename):
236
"""Return the contents of the schema at filename as a StringIO.
237
238
This method verifies that the file is ascii encoded and that no
239
section name is redefined.
240
"""
241
raw_schema = read_content(filename)
242
# Verify that the string is ascii.
243
raw_schema.encode('ascii', 'ignore')
244
# Verify that no sections are redefined.
245
section_names = []
246
for section_name in re.findall(r'^\s*\[[^\]]+\]', raw_schema, re.M):
247
if section_name in section_names:
248
raise RedefinedSectionError(section_name)
249
else:
250
section_names.append(section_name)
251
return StringIO.StringIO(raw_schema)
252
253
def _setSectionSchemasAndCategoryNames(self, parser):
254
"""Set the SectionSchemas and category_names from the config."""
255
category_names = set()
256
templates = {}
257
# Retrieve all the templates first because section() does not
258
# follow the order of the conf file.
259
for name in parser.sections():
260
(section_name, category_name,
261
is_template, is_optional) = self._parseSectionName(name)
262
if is_template:
263
templates[category_name] = dict(parser.items(name))
264
for name in parser.sections():
265
(section_name, category_name,
266
is_template, is_optional) = self._parseSectionName(name)
267
if is_template:
268
continue
269
options = dict(templates.get(category_name, {}))
270
options.update(parser.items(name))
271
self._section_schemas[section_name] = SectionSchema(
272
section_name, options, is_optional)
273
if category_name is not None:
274
category_names.add(category_name)
275
self._category_names = list(category_names)
276
277
_section_name_pattern = re.compile(r'\w[\w.-]+\w')
278
279
def _parseSectionName(self, name):
280
"""Return a 4-tuple of names and kinds embedded in the name.
281
282
:return: (section_name, category_name, is_template, is_optional).
283
section_name is always a string. category_name is a string or
284
None if there is no prefix. is_template and is_optional
285
are False by default, but will be true if the name's suffix
286
ends in '.template' or '.optional'.
287
"""
288
name_parts = name.split('.')
289
is_template = name_parts[-1] == 'template'
290
is_optional = name_parts[-1] == 'optional'
291
if is_template or is_optional:
292
# The suffix is not a part of the section name.
293
# Example: [name.optional] or [category.template]
294
del name_parts[-1]
295
count = len(name_parts)
296
if count == 1 and is_template:
297
# Example: [category.template]
298
category_name = name_parts[0]
299
section_name = name_parts[0]
300
elif count == 1:
301
# Example: [name]
302
category_name = None
303
section_name = name_parts[0]
304
elif count == 2:
305
# Example: [category.name]
306
category_name = name_parts[0]
307
section_name = '.'.join(name_parts)
308
else:
309
raise InvalidSectionNameError('[%s] has too many parts.' % name)
310
if self._section_name_pattern.match(section_name) is None:
311
raise InvalidSectionNameError(
312
'[%s] name does not match [\w.-]+.' % name)
313
return (section_name, category_name, is_template, is_optional)
314
315
@property
316
def section_factory(self):
317
"""See `IConfigSchema`."""
318
return self._section_factory
319
320
@property
321
def category_names(self):
322
"""See `IConfigSchema`."""
323
return self._category_names
324
325
def __iter__(self):
326
"""See `IConfigSchema`."""
327
return self._section_schemas.itervalues()
328
329
def __contains__(self, name):
330
"""See `IConfigSchema`."""
331
return name in self._section_schemas.keys()
332
333
def __getitem__(self, name):
334
"""See `IConfigSchema`."""
335
try:
336
return self._section_schemas[name]
337
except KeyError:
338
raise NoSectionError(name)
339
340
def getByCategory(self, name):
341
"""See `IConfigSchema`."""
342
if name not in self.category_names:
343
raise NoCategoryError(name)
344
section_schemas = []
345
for key in self._section_schemas:
346
section = self._section_schemas[key]
347
category, dummy = section.category_and_section_names
348
if name == category:
349
section_schemas.append(section)
350
return section_schemas
351
352
def _getRequiredSections(self):
353
"""return a dict of `Section`s from the required `SectionSchemas`."""
354
sections = {}
355
for section_schema in self:
356
if not section_schema.optional:
357
sections[section_schema.name] = self.section_factory(
358
section_schema)
359
return sections
360
361
def load(self, filename):
362
"""See `IConfigLoader`."""
363
conf_data = read_content(filename)
364
return self._load(filename, conf_data)
365
366
def loadFile(self, source_file, filename=None):
367
"""See `IConfigLoader`."""
368
conf_data = source_file.read()
369
if filename is None:
370
filename = getattr(source_file, 'name')
371
assert filename is not None, (
372
'filename must be provided if the file-like object '
373
'does not have a name attribute.')
374
return self._load(filename, conf_data)
375
376
def _load(self, filename, conf_data):
377
"""Return a Config parsed from conf_data."""
378
config = Config(self)
379
config.push(filename, conf_data)
380
return config
381
382
383
class ImplicitTypeSchema(ConfigSchema):
384
"""See `IConfigSchema`.
385
386
ImplicitTypeSchema creates a config that supports implicit datatyping
387
of section key values.
388
"""
389
390
_section_factory = ImplicitTypeSection
391
392
393
class ConfigData:
394
"""See `IConfigData`."""
395
implements(IConfigData)
396
397
def __init__(self, filename, sections, extends=None, errors=None):
398
"""Set the configuration data."""
399
self.filename = filename
400
self.name = basename(filename)
401
self._sections = sections
402
self._category_names = self._getCategoryNames()
403
self._extends = extends
404
if errors is None:
405
self._errors = []
406
else:
407
self._errors = errors
408
409
def _getCategoryNames(self):
410
"""Return a tuple of category names that the `Section`s belong to."""
411
category_names = set()
412
for section_name in self._sections:
413
section = self._sections[section_name]
414
category, dummy = section.category_and_section_names
415
if category is not None:
416
category_names.add(category)
417
return tuple(category_names)
418
419
@property
420
def category_names(self):
421
"""See `IConfigData`."""
422
return self._category_names
423
424
def __iter__(self):
425
"""See `IConfigData`."""
426
return self._sections.itervalues()
427
428
def __contains__(self, name):
429
"""See `IConfigData`."""
430
return name in self._sections.keys()
431
432
def __getitem__(self, name):
433
"""See `IConfigData`."""
434
try:
435
return self._sections[name]
436
except KeyError:
437
raise NoSectionError(name)
438
439
def getByCategory(self, name):
440
"""See `IConfigData`."""
441
if name not in self.category_names:
442
raise NoCategoryError(name)
443
sections = []
444
for key in self._sections:
445
section = self._sections[key]
446
category, dummy = section.category_and_section_names
447
if name == category:
448
sections.append(section)
449
return sections
450
451
452
class Config:
453
"""See `IStackableConfig`."""
454
# LAZR config classes may access ConfigData private data.
455
# pylint: disable-msg=W0212
456
implements(IStackableConfig)
457
decorates(IConfigData, context='data')
458
459
def __init__(self, schema):
460
"""Set the schema and configuration."""
461
self._overlays = (
462
ConfigData(schema.filename, schema._getRequiredSections()), )
463
self.schema = schema
464
465
def __getattr__(self, name):
466
"""See `IStackableConfig`."""
467
if name in self.data._sections:
468
return self.data._sections[name]
469
elif name in self.data._category_names:
470
return Category(name, self.data.getByCategory(name))
471
raise AttributeError("No section or category named %s." % name)
472
473
@property
474
def data(self):
475
"""See `IStackableConfig`."""
476
return self.overlays[0]
477
478
@property
479
def extends(self):
480
"""See `IStackableConfig`."""
481
if len(self.overlays) == 1:
482
# The ConfigData made from the schema defaults extends nothing.
483
return None
484
else:
485
return self.overlays[1]
486
487
@property
488
def overlays(self):
489
"""See `IStackableConfig`."""
490
return self._overlays
491
492
def validate(self):
493
"""See `IConfigData`."""
494
if len(self.data._errors) > 0:
495
message = "%s is not valid." % self.name
496
raise ConfigErrors(message, errors=self.data._errors)
497
return True
498
499
def push(self, conf_name, conf_data):
500
"""See `IStackableConfig`.
501
502
Create a new ConfigData object from the raw conf_data, and
503
place it on top of the overlay stack. If the conf_data extends
504
another conf, a ConfigData object will be created for that first.
505
"""
506
conf_data = dedent(conf_data)
507
confs = self._getExtendedConfs(conf_name, conf_data)
508
confs.reverse()
509
for conf_name, parser, encoding_errors in confs:
510
if self.data.filename == self.schema.filename == conf_name:
511
# Do not parse the schema file twice in a row.
512
continue
513
config_data = self._createConfigData(
514
conf_name, parser, encoding_errors)
515
self._overlays = (config_data, ) + self._overlays
516
517
def _getExtendedConfs(self, conf_filename, conf_data, confs=None):
518
"""Return a list of 3-tuple(conf_name, parser, encoding_errors).
519
520
:param conf_filename: The path and name the conf file.
521
:param conf_data: Unparsed config data.
522
:param confs: A list of confs that extend filename.
523
:return: A list of confs ordered from extender to extendee.
524
:raises IOError: If filename cannot be read.
525
526
This method parses the config data and checks for encoding errors.
527
It checks parsed config data for the extends key in the meta section.
528
It reads the unparsed config_data from the extended filename.
529
It passes filename, data, and the working list to itself.
530
"""
531
if confs is None:
532
confs = []
533
encoding_errors = self._verifyEncoding(conf_data)
534
parser = RawConfigParser()
535
parser.readfp(StringIO.StringIO(conf_data), conf_filename)
536
confs.append((conf_filename, parser, encoding_errors))
537
if parser.has_option('meta', 'extends'):
538
base_path = dirname(conf_filename)
539
extends_name = parser.get('meta', 'extends')
540
extends_filename = abspath('%s/%s' % (base_path, extends_name))
541
extends_data = read_content(extends_filename)
542
self._getExtendedConfs(extends_filename, extends_data, confs)
543
return confs
544
545
def _createConfigData(self, conf_name, parser, encoding_errors):
546
"""Return a new ConfigData object created from a parsed conf file.
547
548
:param conf_name: the full name of the config file, may be a filepath.
549
:param parser: the parsed config file; an instance of ConfigParser.
550
:param encoding_errors: a list of encoding error in the config file.
551
:return: a new ConfigData object.
552
553
This method extracts the sections, keys, and values from the parser
554
to construct a new ConfigData object The list of encoding errors are
555
incorporated into the the list of data-related errors for the
556
ConfigData.
557
"""
558
sections = {}
559
for section in self.data:
560
sections[section.name] = section.clone()
561
errors = list(self.data._errors)
562
errors.extend(encoding_errors)
563
extends = None
564
for section_name in parser.sections():
565
if section_name == 'meta':
566
extends, meta_errors = self._loadMetaData(parser)
567
errors.extend(meta_errors)
568
continue
569
if (section_name.endswith('.template')
570
or section_name.endswith('.optional')):
571
# This section is a schema directive.
572
continue
573
if section_name not in self.schema:
574
# Any section not in the the schema is an error.
575
msg = "%s does not have a %s section." % (
576
self.schema.name, section_name)
577
errors.append(UnknownSectionError(msg))
578
continue
579
if section_name not in self.data:
580
# Create the optional section from the schema.
581
section_schema = self.schema[section_name]
582
sections[section_name] = self.schema.section_factory(
583
section_schema)
584
# Update the section with the parser options.
585
items = parser.items(section_name)
586
section_errors = sections[section_name].update(items)
587
errors.extend(section_errors)
588
return ConfigData(conf_name, sections, extends, errors)
589
590
def _verifyEncoding(self, config_data):
591
"""Verify that the data is ASCII encoded.
592
593
:return: a list of UnicodeDecodeError errors. If there are no
594
errors, return an empty list.
595
"""
596
errors = []
597
try:
598
config_data.encode('ascii', 'ignore')
599
except UnicodeDecodeError, error:
600
errors.append(error)
601
return errors
602
603
def _loadMetaData(self, parser):
604
"""Load the config meta data from the ConfigParser.
605
606
The meta section is reserved for the LAZR config parser.
607
608
:return: a list of errors if there are errors, or an empty list.
609
"""
610
extends = None
611
errors = []
612
for key in parser.options('meta'):
613
if key == "extends":
614
extends = parser.get('meta', 'extends')
615
else:
616
# Any other key is an error.
617
msg = "The meta section does not have a %s key." % key
618
errors.append(UnknownKeyError(msg))
619
return (extends, errors)
620
621
def pop(self, conf_name):
622
"""See `IStackableConfig`."""
623
index = self._getIndexOfOverlay(conf_name)
624
removed_overlays = self.overlays[:index]
625
self._overlays = self.overlays[index:]
626
return removed_overlays
627
628
def _getIndexOfOverlay(self, conf_name):
629
"""Return the index of the config named conf_name.
630
631
The bottom of the stack cannot never be returned because it was
632
made from the schema.
633
"""
634
schema_index = len(self.overlays) - 1
635
for index, config_data in enumerate(self.overlays):
636
if index == schema_index and config_data.name == conf_name:
637
raise NoConfigError("Cannot pop the schema's default config.")
638
if config_data.name == conf_name:
639
return index + 1
640
# The config data was not found in the overlays.
641
raise NoConfigError('No config with name: %s.' % conf_name)
642
643
644
class Category:
645
"""See `ICategory`."""
646
implements(ICategory)
647
648
def __init__(self, name, sections):
649
"""Initialize the Category its name and a list of sections."""
650
self.name = name
651
self._sections = {}
652
for section in sections:
653
self._sections[section.name] = section
654
655
def __getattr__(self, name):
656
"""See `ICategory`."""
657
full_name = "%s.%s" % (self.name, name)
658
if full_name in self._sections:
659
return self._sections[full_name]
660
raise AttributeError("No section named %s." % name)
661
662
663
def as_host_port(value, default_host='localhost', default_port=25):
664
"""Return a 2-tuple of (host, port) from a value like 'host:port'.
665
666
:param value: The configuration value.
667
:type value: string
668
:param default_host: Optional host name to use if the configuration value
669
is missing the host name.
670
:type default_host: string
671
:param default_port: Optional port number to use if the configuration
672
value is missing the port number.
673
:type default_port: integer
674
:return: a 2-tuple of the form (host, port)
675
:rtype: 2-tuple of (string, integer)
676
"""
677
if ':' in value:
678
host, port = value.split(':')
679
if host == '':
680
host = default_host
681
port = int(port)
682
else:
683
host = value
684
port = default_port
685
return host, port
686
687
688
def as_username_groupname(value=None):
689
"""Turn a string of the form user:group into the user and group names.
690
691
:param value: The configuration value.
692
:type value: a string containing exactly one colon, or None
693
:return: a 2-tuple of (username, groupname). If `value` was None, then
694
the current user and group names are returned.
695
:rtype: 2-tuple of type (string, string)
696
"""
697
if value:
698
user, group = value.split(':', 1)
699
else:
700
user = pwd.getpwuid(os.getuid()).pw_name
701
group = grp.getgrgid(os.getgid()).gr_name
702
return user, group
703
704
705
def _sort_order(a, b):
706
"""Sort timedelta suffixes from greatest to least."""
707
if len(a) == 0:
708
return -1
709
if len(b) == 0:
710
return 1
711
order = dict(
712
w=0, # weeks
713
d=1, # days
714
h=2, # hours
715
m=3, # minutes
716
s=4, # seconds
717
)
718
suffix_a = order.get(a[-1])
719
suffix_b = order.get(b[-1])
720
if suffix_a is None or suffix_b is None:
721
raise ValueError
722
return cmp(suffix_a, suffix_b)
723
724
725
def as_timedelta(value):
726
"""Convert a value string to the equivalent timedeta."""
727
# Technically, the regex will match multiple decimal points in the
728
# left-hand side, but that's okay because the float/int conversion below
729
# will properly complain if there's more than one dot.
730
components = sorted(re.findall(r'([\d.]+[smhdw])', value),
731
cmp=_sort_order)
732
# Complain if the components are out of order.
733
if ''.join(components) != value:
734
raise ValueError
735
keywords = dict((interval[0].lower(), interval)
736
for interval in ('weeks', 'days', 'hours',
737
'minutes', 'seconds'))
738
keyword_arguments = {}
739
for interval in components:
740
if len(interval) == 0:
741
raise ValueError
742
keyword = keywords.get(interval[-1].lower())
743
if keyword is None:
744
raise ValueError
745
if keyword in keyword_arguments:
746
raise ValueError
747
if '.' in interval[:-1]:
748
converted = float(interval[:-1])
749
else:
750
converted = int(interval[:-1])
751
keyword_arguments[keyword] = converted
752
if len(keyword_arguments) == 0:
753
raise ValueError
754
return datetime.timedelta(**keyword_arguments)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1