3
# Natural Language Toolkit: Documentation generation script
5
# Copyright (C) 2001-2006 University of Pennsylvania
6
# Author: Edward Loper <edloper@gradient.cis.upenn.edu>
7
# Steven Bird (substantially cut down)
8
# URL: <http://nltk.sf.net>
9
# For license information, see LICENSE.TXT
12
This is a customized driver for converting docutils reStructuredText
13
documents into HTML and LaTeX. It customizes the standard writers in
16
- Source code highlighting is added to all doctest blocks. In
17
the HTML output, highlighting is performed using css classes:
18
'pysrc-prompt', 'pysrc-keyword', 'pysrc-string', 'pysrc-comment',
21
import re, os.path, textwrap, sys, pickle
22
from optparse import OptionParser
24
import docutils.core, docutils.nodes, docutils.io
25
from docutils.writers import Writer
26
from docutils.writers.html4css1 import HTMLTranslator, Writer as HTMLWriter
27
from docutils.parsers.rst import directives, roles
28
from docutils.readers.standalone import Reader as StandaloneReader
29
from docutils.transforms import Transform
30
import docutils.writers.html4css1
31
from doctest import DocTestParser
32
import docutils.statemachine
36
LATEX_VALIGN_IS_BROKEN = True
37
"""Set to true to compensate for a bug in the latex writer. I've
38
submitted a patch to docutils, so hopefully this wil be fixed
42
"""The scaling factor that should be used to display bitmapped images
43
in latex/pdf output (specified in dots per inch). E.g., if a
44
bitmapped image is 100 pixels wide, it will be scaled to
45
100/LATEX_DPI inches wide for the latex/pdf output. (Larger
46
values produce smaller images in the generated pdf.)"""
48
TREE_IMAGE_DIR = 'tree_images/'
49
"""The directory that tree images should be written to."""
51
EXTERN_REFERENCE_FILES = []
52
"""A list of .ref files, for crossrefering to external documents (used
53
when building one chapter at a time)."""
55
BIBTEX_FILE = '../refs.bib'
56
"""The name of the bibtex file used to generate bibliographic entries."""
58
BIBLIOGRAPHY_HTML = "bibliography.html"
59
"""The name of the HTML file containing the bibliography (for
60
hyperrefs from citations)."""
62
# needs to include "../doc" so it works in /doc_contrib
63
LATEX_STYLESHEET_PATH = '../../doc/definitions.sty'
64
"""The name of the LaTeX style file used for generating PDF output."""
66
LOCAL_BIBLIOGRAPHY = False
67
"""If true, assume that this document contains the bibliography, and
68
link to it locally; if false, assume that bibliographic links
69
should point to L{BIBLIOGRAPHY_HTML}."""
71
PYLISTING_DIR = 'pylisting/'
72
"""The directory where pylisting files should be written."""
74
PYLISTING_EXTENSION = ".py"
75
"""Extension for pylisting files."""
77
INCLUDE_DOCTESTS_IN_PYLISTING_FILES = False
78
"""If true, include code from doctests in the generated pylisting
81
CALLOUT_IMG = '<img src="callouts/callout%s.gif" alt="[%s]" class="callout" />'
82
"""HTML code for callout images in pylisting blocks."""
84
REF_EXTENSION = '.ref'
85
"""File extension for reference files."""
87
# needs to include "../doc" so it works in /doc_contrib
88
CSS_STYLESHEET = '/dev/null' #/home/nick/exercise-ui/ivle/webapp/tutorial/media/nltkdoc.css'
92
"""A global variable, set by main(), indicating the output format for
93
the current file. Can be 'latex' or 'html' or 'ref'."""
95
OUTPUT_BASENAME = None
96
"""A global variable, set by main(), indicating the base filename
97
of the current file (i.e., the filename with its extension
98
stripped). This is used to generate filenames for images."""
100
COPY_CLIPBOARD_JS = ''
103
######################################################################
105
######################################################################
107
def read_ref_file(basename=None):
108
if basename is None: basename = OUTPUT_BASENAME
109
if not os.path.exists(basename + REF_EXTENSION):
110
warning('File %r does not exist!' %
111
(basename + REF_EXTENSION))
112
return dict(targets=(),terms={},reference_labes={})
113
f = open(basename + REF_EXTENSION)
114
ref_info = pickle.load(f)
118
def write_ref_file(ref_info):
119
f = open(OUTPUT_BASENAME + REF_EXTENSION, 'w')
120
pickle.dump(ref_info, f)
123
def add_to_ref_file(**ref_info):
124
if os.path.exists(OUTPUT_BASENAME + REF_EXTENSION):
125
info = read_ref_file()
126
info.update(ref_info)
129
write_ref_file(ref_info)
131
######################################################################
133
######################################################################
135
class example(docutils.nodes.paragraph): pass
137
def example_directive(name, arguments, options, content, lineno,
138
content_offset, block_text, state, state_machine):
142
.. example:: John went to the store.
144
To refer to examples, use::
147
.. example:: John went to the store.
149
In store_, John performed an action.
151
text = '\n'.join(content)
153
state.nested_parse(content, content_offset, node)
155
example_directive.content = True
156
directives.register_directive('example', example_directive)
157
directives.register_directive('ex', example_directive)
159
def doctest_directive(name, arguments, options, content, lineno,
160
content_offset, block_text, state, state_machine):
162
Used to explicitly mark as doctest blocks things that otherwise
163
wouldn't look like doctest blocks.
165
text = '\n'.join(content)
166
if re.match(r'.*\n\s*\n', block_text):
167
warning('doctest-ignore on line %d will not be ignored, '
168
'because there is\na blank line between ".. doctest-ignore::"'
169
' and the doctest example.' % lineno)
170
return [docutils.nodes.doctest_block(text, text, codeblock=True)]
171
doctest_directive.content = True
172
directives.register_directive('doctest-ignore', doctest_directive)
175
def tree_directive(name, arguments, options, content, lineno,
176
content_offset, block_text, state, state_machine):
178
text = '\n'.join(arguments) + '\n'.join(content)
180
# Note: the two filenames generated by these two cases should be
181
# different, to prevent conflicts.
182
if OUTPUT_FORMAT == 'latex':
183
density, scale = 300, 150
184
scale = scale * options.get('scale', 100) / 100
185
filename = '%s-tree-%s.pdf' % (OUTPUT_BASENAME, _treenum)
186
align = LATEX_VALIGN_IS_BROKEN and 'bottom' or 'top'
187
elif OUTPUT_FORMAT == 'html':
188
density, scale = 100, 100
189
density = density * options.get('scale', 100) / 100
190
filename = '%s-tree-%s.png' % (OUTPUT_BASENAME, _treenum)
192
elif OUTPUT_FORMAT == 'ref':
195
assert 0, 'bad output format %r' % OUTPUT_FORMAT
196
if not os.path.exists(TREE_IMAGE_DIR):
197
os.mkdir(TREE_IMAGE_DIR)
199
filename = os.path.join(TREE_IMAGE_DIR, filename)
200
tree_to_image(text, filename, density)
203
warning('Error parsing tree: %s\n%s\n%s' % (e, text, filename))
204
return [example(text, text)]
206
imagenode = docutils.nodes.image(uri=filename, scale=scale, align=align)
209
tree_directive.arguments = (1,0,1)
210
tree_directive.content = True
211
tree_directive.options = {'scale': directives.nonnegative_int}
212
directives.register_directive('tree', tree_directive)
214
def avm_directive(name, arguments, options, content, lineno,
215
content_offset, block_text, state, state_machine):
216
text = '\n'.join(content)
218
if OUTPUT_FORMAT == 'latex':
219
latex_avm = parse_avm(textwrap.dedent(text)).as_latex()
220
return [docutils.nodes.paragraph('','',
221
docutils.nodes.raw('', latex_avm, format='latex'))]
222
elif OUTPUT_FORMAT == 'html':
223
return [parse_avm(textwrap.dedent(text)).as_table()]
224
elif OUTPUT_FORMAT == 'ref':
225
return [docutils.nodes.paragraph()]
226
except ValueError, e:
227
if isinstance(e.args[0], int):
228
warning('Error parsing avm on line %s' % (lineno+e.args[0]))
231
warning('Error parsing avm on line %s: %s' % (lineno, e))
232
node = example(text, text)
233
state.nested_parse(content, content_offset, node)
235
avm_directive.content = True
236
directives.register_directive('avm', avm_directive)
238
def def_directive(name, arguments, options, content, lineno,
239
content_offset, block_text, state, state_machine):
240
state_machine.document.setdefault('__defs__', {})[arguments[0]] = 1
242
def_directive.arguments = (1, 0, 0)
243
directives.register_directive('def', def_directive)
245
def ifdef_directive(name, arguments, options, content, lineno,
246
content_offset, block_text, state, state_machine):
247
if arguments[0] in state_machine.document.get('__defs__', ()):
248
node = docutils.nodes.compound('')
249
state.nested_parse(content, content_offset, node)
253
ifdef_directive.arguments = (1, 0, 0)
254
ifdef_directive.content = True
255
directives.register_directive('ifdef', ifdef_directive)
257
def ifndef_directive(name, arguments, options, content, lineno,
258
content_offset, block_text, state, state_machine):
259
if arguments[0] not in state_machine.document.get('__defs__', ()):
260
node = docutils.nodes.compound('')
261
state.nested_parse(content, content_offset, node)
265
ifndef_directive.arguments = (1, 0, 0)
266
ifndef_directive.content = True
267
directives.register_directive('ifndef', ifndef_directive)
270
######################################################################
272
######################################################################
274
def table_directive(name, arguments, options, content, lineno,
275
content_offset, block_text, state, state_machine):
276
# The identifier for this table.
278
table_id = arguments[0]
279
if table_id in _table_ids:
280
warning("Duplicate table id %r" % table_id)
281
_table_ids.add(table_id)
283
# Create a target element for the table
284
target = docutils.nodes.target(names=[table_id])
285
state_machine.document.note_explicit_target(target)
287
# Parse the contents.
288
node = docutils.nodes.compound('')
289
state.nested_parse(content, content_offset, node)
290
if len(node) == 0 or not isinstance(node[0], docutils.nodes.table):
291
return [state_machine.reporter.error(
292
'Error in "%s" directive: expected table as first child' %
295
# Move the caption into the table.
297
caption = docutils.nodes.caption('','', *node[1:])
298
table.append(caption)
300
# Return the target and the table.
302
return [target, table]
307
table_directive.arguments = (0,1,0) # 1 optional arg, no whitespace
308
table_directive.content = True
309
table_directive.options = {'caption': directives.unchanged}
310
directives.register_directive('table', table_directive)
313
######################################################################
315
######################################################################
316
# We define a new attribute for doctest blocks: 'is_codeblock'. If
317
# this attribute is true, then the block contains python code only
318
# (i.e., don't expect to find prompts.)
320
class pylisting(docutils.nodes.General, docutils.nodes.Element):
322
Python source code listing.
324
Children: doctest_block+ caption?
326
class callout_marker(docutils.nodes.Inline, docutils.nodes.Element):
328
A callout marker for doctest block. This element contains no
329
children; and defines the attribute 'number'.
332
DOCTEST_BLOCK_RE = re.compile('((?:[ ]*>>>.*\n?(?:.*[^ ].*\n?)+\s*)+)',
334
CALLOUT_RE = re.compile(r'#[ ]+\[_([\w-]+)\][ ]*$', re.MULTILINE)
336
from docutils.nodes import fully_normalize_name as normalize_name
339
def pylisting_directive(name, arguments, options, content, lineno,
340
content_offset, block_text, state, state_machine):
341
# The identifier for this listing.
342
listing_id = arguments[0]
343
if listing_id in _listing_ids:
344
warning("Duplicate listing id %r" % listing_id)
345
_listing_ids.add(listing_id)
347
# Create the pylisting element itself.
348
listing = pylisting('\n'.join(content), name=listing_id, callouts={})
350
# Create a target element for the pylisting.
351
target = docutils.nodes.target(names=[listing_id])
352
state_machine.document.note_explicit_target(target)
354
# Divide the text into doctest blocks.
355
for i, v in enumerate(DOCTEST_BLOCK_RE.split('\n'.join(content))):
356
pysrc = re.sub(r'\A( *\n)+', '', v.rstrip())
358
listing.append(docutils.nodes.doctest_block(pysrc, pysrc,
359
is_codeblock=(i%2==0)))
361
# Add an optional caption.
362
if options.get('caption'):
363
cap = options['caption'].split('\n')
364
caption = docutils.nodes.compound()
365
state.nested_parse(docutils.statemachine.StringList(cap),
366
content_offset, caption)
367
if (len(caption) == 1 and isinstance(caption[0],
368
docutils.nodes.paragraph)):
369
listing.append(docutils.nodes.caption('', '', *caption[0]))
371
warning("Caption should be a single paragraph")
372
listing.append(docutils.nodes.caption('', '', *caption))
374
return [target, listing]
376
pylisting_directive.arguments = (1,0,0) # 1 required arg, no whitespace
377
pylisting_directive.content = True
378
pylisting_directive.options = {'caption': directives.unchanged}
379
directives.register_directive('pylisting', pylisting_directive)
381
def callout_directive(name, arguments, options, content, lineno,
382
content_offset, block_text, state, state_machine):
384
prefix = '%s-' % arguments[0]
387
node = docutils.nodes.compound('')
388
state.nested_parse(content, content_offset, node)
389
if not (len(node.children) == 1 and
390
isinstance(node[0], docutils.nodes.field_list)):
391
return [state_machine.reporter.error(
392
'Error in "%s" directive: may contain a single defintion '
393
'list only.' % (name), line=lineno)]
395
node[0]['classes'] = ['callouts']
396
for field in node[0]:
397
if len(field[0]) != 1:
398
return [state_machine.reporter.error(
399
'Error in "%s" directive: bad field id' % (name), line=lineno)]
401
field_name = prefix+('%s' % field[0][0])
403
field[0].append(docutils.nodes.reference(field_name, field_name,
405
field[0]['classes'] = ['callout']
409
callout_directive.arguments = (0,1,0) # 1 optional arg, no whitespace
410
callout_directive.content = True
411
directives.register_directive('callouts', callout_directive)
413
_OPTION_DIRECTIVE_RE = re.compile(
414
r'(\n[ ]*\.\.\.[ ]*)?#\s*doctest:\s*([^\n\'"]*)$', re.MULTILINE)
415
def strip_doctest_directives(text):
416
return _OPTION_DIRECTIVE_RE.sub('', text)
419
######################################################################
421
######################################################################
423
def rst_example_directive(name, arguments, options, content, lineno,
424
content_offset, block_text, state, state_machine):
425
raw = docutils.nodes.literal_block('', '\n'.join(content))
426
out = docutils.nodes.compound('')
427
state.nested_parse(content, content_offset, out)
428
if OUTPUT_FORMAT == 'latex':
430
docutils.nodes.definition_list('',
431
docutils.nodes.definition_list_item('',
432
docutils.nodes.term('','Input'),
433
docutils.nodes.definition('', raw)),
434
docutils.nodes.definition_list_item('',
435
docutils.nodes.term('','Rendered'),
436
docutils.nodes.definition('', out)))]
439
docutils.nodes.table('',
440
docutils.nodes.tgroup('',
441
docutils.nodes.colspec(colwidth=5,classes=['rst-raw']),
442
docutils.nodes.colspec(colwidth=5),
443
docutils.nodes.thead('',
444
docutils.nodes.row('',
445
docutils.nodes.entry('',
446
docutils.nodes.paragraph('','Input')),
447
docutils.nodes.entry('',
448
docutils.nodes.paragraph('','Rendered')))),
449
docutils.nodes.tbody('',
450
docutils.nodes.row('',
451
docutils.nodes.entry('',raw),
452
docutils.nodes.entry('',out)))),
453
classes=["rst-example"])]
455
rst_example_directive.arguments = (0, 0, 0)
456
rst_example_directive.content = True
457
directives.register_directive('rst_example', rst_example_directive)
460
######################################################################
462
######################################################################
466
This | is | used | to | make | aligned | glosses.
467
NN | BE | VB | TO | VB | JJ | NN
471
class gloss(docutils.nodes.Element): "glossrow+"
472
class glossrow(docutils.nodes.Element): "paragraph+"
474
def gloss_directive(name, arguments, options, content, lineno,
475
content_offset, block_text, state, state_machine):
476
# Transform into a table.
477
lines = list(content)
478
maxlen = max(len(line) for line in lines)
479
lines = [('|%-'+`maxlen`+'s|') % line for line in lines]
482
for line in (lines+['']):
483
div = ['-']*(maxlen+2)
484
for m in re.finditer(r'\|', prevline):
486
for m in re.finditer(r'\|', line):
488
tablestr += ''.join(div) + '\n' + line + '\n'
490
table_lines = tablestr.strip().split('\n')
491
new_content = docutils.statemachine.StringList(table_lines)
492
# [XX] DEBUG GLOSSES:
493
# print 'converted to:'
497
node = docutils.nodes.compound('')
498
state.nested_parse(new_content, content_offset, node)
499
if not (len(node.children) == 1 and
500
isinstance(node[0], docutils.nodes.table)):
501
error = state_machine.reporter.error(
502
'Error in "%s" directive: may contain a single table '
503
'only.' % (name), line=lineno)
506
table['classes'] = ['gloss', 'nolines']
509
for colspec in colspecs:
510
colspec['colwidth'] = colspec.get('colwidth',4)/2
512
return [example('', '', table)]
513
gloss_directive.arguments = (0, 0, 0)
514
gloss_directive.content = True
515
directives.register_directive('gloss', gloss_directive)
518
######################################################################
520
######################################################################
522
class Citations(Transform):
523
default_priority = 500 # before footnotes.
525
if not os.path.exists(BIBTEX_FILE):
526
warning('Warning bibtex file %r not found. '
527
'Not linking citations.' % BIBTEX_FILE)
529
bibliography = self.read_bibinfo(BIBTEX_FILE)
530
for k, citation_refs in self.document.citation_refs.items():
531
for citation_ref in citation_refs[:]:
532
cite = bibliography.get(citation_ref['refname'].lower())
534
new_cite = self.citeref(cite, citation_ref['refname'])
535
citation_ref.replace_self(new_cite)
536
self.document.citation_refs[k].remove(citation_ref)
538
def citeref(self, cite, key):
539
if LOCAL_BIBLIOGRAPHY:
540
return docutils.nodes.raw('', '\cite{%s}' % key, format='latex')
542
return docutils.nodes.reference('', '', docutils.nodes.Text(cite),
543
refuri='%s#%s' % (BIBLIOGRAPHY_HTML, key))
545
BIB_ENTRY = re.compile(r'@\w+{.*')
546
def read_bibinfo(self, filename):
547
bibliography = {} # key -> authors, year
549
for line in open(filename):
552
# @InProceedings{<key>,
553
m = re.match(r'@\w+{([^,]+),$', line)
555
key = m.group(1).strip().lower()
556
bibliography[key] = [None, None]
558
# author = <authors>,
559
m = re.match(r'(?i)author\s*=\s*(.*)$', line)
561
bibliography[key][0] = self.bib_authors(m.group(1))
563
m = re.match(r'(?i)editor\s*=\s*(.*)$', line)
565
bibliography[key][0] = self.bib_authors(m.group(1))
568
m = re.match(r'(?i)year\s*=\s*(.*)$', line)
570
bibliography[key][1] = self.bib_year(m.group(1))
571
for key in bibliography:
572
if bibliography[key][0] is None: warning('no author found:', key)
573
if bibliography[key][1] is None: warning('no year found:', key)
574
bibliography[key] = '[%s, %s]' % tuple(bibliography[key])
575
#debug('%20s %s' % (key, `bibliography[key]`))
578
def bib_year(self, year):
579
return re.sub(r'["\'{},]', "", year)
581
def bib_authors(self, authors):
582
# Strip trailing comma:
583
if authors[-1:] == ',': authors=authors[:-1]
584
# Strip quotes or braces:
585
authors = re.sub(r'"(.*)"$', r'\1', authors)
586
authors = re.sub(r'{(.*)}$', r'\1', authors)
587
authors = re.sub(r"'(.*)'$", r'\1', authors)
589
authors = re.split(r'\s+and\s+', authors)
590
# Keep last name only:
591
authors = [a.split()[-1] for a in authors]
593
if len(authors) == 1:
595
elif len(authors) == 2:
596
return '%s & %s' % tuple(authors)
597
elif len(authors) == 3:
598
return '%s, %s, & %s' % tuple(authors)
600
return '%s et al' % authors[0]
604
######################################################################
606
######################################################################
607
class termdef(docutils.nodes.Inline, docutils.nodes.TextElement): pass
608
class idxterm(docutils.nodes.Inline, docutils.nodes.TextElement): pass
609
class index(docutils.nodes.Element): pass
611
def idxterm_role(name, rawtext, text, lineno, inliner,
612
options={}, content=[]):
613
if name == 'dt': options['classes'] = ['termdef']
614
elif name == 'topic': options['classes'] = ['topic']
615
else: options['classes'] = ['term']
616
# Recursively parse the contents of the index term, in case it
617
# contains a substitiution (like |alpha|).
618
nodes, msgs = inliner.parse(text, lineno, memo=inliner,
619
parent=inliner.parent)
620
return [idxterm(rawtext, '', *nodes, **options)], []
622
roles.register_canonical_role('dt', idxterm_role)
623
roles.register_canonical_role('idx', idxterm_role)
624
roles.register_canonical_role('topic', idxterm_role)
626
def index_directive(name, arguments, options, content, lineno,
627
content_offset, block_text, state, state_machine):
628
pending = docutils.nodes.pending(ConstructIndex)
629
pending.details.update(options)
630
state_machine.document.note_pending(pending)
631
return [index('', pending)]
632
index_directive.arguments = (0, 0, 0)
633
index_directive.content = False
634
index_directive.options = {'extern': directives.flag}
635
directives.register_directive('index', index_directive)
638
class SaveIndexTerms(Transform):
639
default_priority = 810 # before NumberReferences transform
641
v = FindTermVisitor(self.document)
642
self.document.walkabout(v)
644
if OUTPUT_FORMAT == 'ref':
645
add_to_ref_file(terms=v.terms)
647
class ConstructIndex(Transform):
648
default_priority = 820 # after NumberNodes, before NumberReferences.
650
# Find any indexed terms in this document.
651
v = FindTermVisitor(self.document)
652
self.document.walkabout(v)
655
# Check the extern reference files for additional terms.
656
if 'extern' in self.startnode.details:
657
for filename in EXTERN_REFERENCE_FILES:
658
basename = os.path.splitext(filename)[0]
659
terms.update(read_ref_file(basename)['terms'])
661
# Build the index & insert it into the document.
662
index_node = self.build_index(terms)
663
self.startnode.replace_self(index_node)
665
def build_index(self, terms):
666
if not terms: return []
668
top = docutils.nodes.bullet_list('', classes=['index'])
672
for key in sorted(terms.keys()):
673
if key[:1] != start_letter:
674
top.append(docutils.nodes.list_item(
675
'', docutils.nodes.paragraph('', key[:1].upper()+'\n',
676
classes=['index-heading']),
677
docutils.nodes.bullet_list('', classes=['index-section']),
679
section = top[-1][-1]
680
section.append(self.entry(terms[key]))
681
start_letter = key[:1]
685
def entry(self, term_info):
686
entrytext, name, sectnum = term_info
687
if sectnum is not None:
688
entrytext.append(docutils.nodes.emphasis('', ' (%s)' % sectnum))
689
ref = docutils.nodes.reference('', '', refid=name,
692
para = docutils.nodes.paragraph('', '', ref)
693
return docutils.nodes.list_item('', para, classes=['index'])
695
class FindTermVisitor(docutils.nodes.SparseNodeVisitor):
696
def __init__(self, document):
698
docutils.nodes.NodeVisitor.__init__(self, document)
699
def unknown_visit(self, node): pass
700
def unknown_departure(self, node): pass
702
def visit_idxterm(self, node):
703
node['name'] = node['id'] = self.idxterm_key(node)
704
node['names'] = node['ids'] = [node['id']]
705
container = self.container_section(node)
707
entrytext = node.deepcopy()
708
if container: sectnum = container.get('sectnum')
711
self.terms[node['name']] = (entrytext, name, sectnum)
713
def idxterm_key(self, node):
714
key = re.sub('\W', '_', node.astext().lower())+'_index_term'
715
if key not in self.terms: return key
717
while '%s_%d' % (key, n) in self.terms: n += 1
718
return '%s_%d' % (key, n)
720
def container_section(self, node):
721
while not isinstance(node, docutils.nodes.section):
722
if node.parent is None: return None
723
else: node = node.parent
728
######################################################################
730
######################################################################
732
class ResolveExternalCrossrefs(Transform):
734
Using the information from EXTERN_REFERENCE_FILES, look for any
735
links to external targets, and set their `refuid` appropriately.
736
Also, if they are a figure, section, table, or example, then
737
replace the link of the text with the appropriate counter.
739
default_priority = 849 # right before dangling refs
742
ref_dict = self.build_ref_dict()
743
v = ExternalCrossrefVisitor(self.document, ref_dict)
744
self.document.walkabout(v)
746
def build_ref_dict(self):
747
"""{target -> (uri, label)}"""
749
for filename in EXTERN_REFERENCE_FILES:
750
basename = os.path.splitext(filename)[0]
751
if OUTPUT_FORMAT == 'html':
752
uri = os.path.split(basename)[-1]+'.html'
754
uri = os.path.split(basename)[-1]+'.pdf'
755
if basename == OUTPUT_BASENAME:
756
pass # don't read our own ref file.
757
elif not os.path.exists(basename+REF_EXTENSION):
758
warning('%s does not exist' % (basename+REF_EXTENSION))
760
ref_info = read_ref_file(basename)
761
for ref in ref_info['targets']:
762
label = ref_info['reference_labels'].get(ref)
763
ref_dict[ref] = (uri, label)
767
class ExternalCrossrefVisitor(docutils.nodes.NodeVisitor):
768
def __init__(self, document, ref_dict):
769
docutils.nodes.NodeVisitor.__init__(self, document)
770
self.ref_dict = ref_dict
771
def unknown_visit(self, node): pass
772
def unknown_departure(self, node): pass
774
# Don't mess with the table of contents.
775
def visit_topic(self, node):
776
if 'contents' in node.get('classes', ()):
777
raise docutils.nodes.SkipNode
779
def visit_reference(self, node):
780
if node.resolved: return
781
node_id = node.get('refid') or node.get('refname')
782
if node_id in self.ref_dict:
783
uri, label = self.ref_dict[node_id]
784
#debug('xref: %20s -> %-30s (label=%s)' % (
785
# node_id, uri+'#'+node_id, label))
786
node['refuri'] = '%s#%s' % (uri, node_id)
789
if label is not None:
790
if node.get('expanded_ref'):
791
warning('Label %s is defined both locally (%s) and '
792
'externally (%s)' % (node_id, node[0], label))
796
node.append(docutils.nodes.Text(label))
797
expand_reference_text(node)
799
######################################################################
801
######################################################################
804
.. exercise:: path.xml
807
class exercise(docutils.nodes.paragraph,docutils.nodes.Element): pass
809
def exercise_directive(name, arguments, options, content, lineno,
810
content_offset, block_text, state, state_machine):
811
return [exercise('', arguments[0])]
813
exercise_directive.arguments = (1, 0, 0)
814
exercise_directive.content = False
815
directives.register_directive('exercise', exercise_directive)
818
######################################################################
819
#{ Challenges (optional exercises; harder than usual)
820
######################################################################
823
.. challenge:: path.xml
826
class challenge(docutils.nodes.paragraph,docutils.nodes.Element): pass
828
def challenge_directive(name, arguments, options, content, lineno,
829
content_offset, block_text, state, state_machine):
830
return [challenge('', arguments[0])]
832
challenge_directive.arguments = (1, 0, 0)
833
challenge_directive.content = False
834
directives.register_directive('challenge', challenge_directive)
838
######################################################################
839
#{ Figure & Example Numbering
840
######################################################################
842
# [xx] number examples, figures, etc, relative to chapter? e.g.,
843
# figure 3.2? maybe number examples within-chapter, but then restart
846
class section_context(docutils.nodes.Invisible, docutils.nodes.Element):
847
def __init__(self, context):
848
docutils.nodes.Element.__init__(self, '', context=context)
849
assert self['context'] in ('body', 'preface', 'appendix')
851
def section_context_directive(name, arguments, options, content, lineno,
852
content_offset, block_text, state, state_machine):
853
return [section_context(name)]
854
section_context_directive.arguments = (0,0,0)
855
directives.register_directive('preface', section_context_directive)
856
directives.register_directive('body', section_context_directive)
857
directives.register_directive('appendix', section_context_directive)
859
class NumberNodes(Transform):
861
This transform adds numbers to figures, tables, and examples; and
862
converts references to the figures, tables, and examples to use
863
these numbers. For example, given the rst source::
866
.. ex:: John likes Mary.
868
See example my_example_.
870
This transform will assign a number to the example, '(1)', and
871
will replace the following text with 'see example (1)', with an
874
# dangling = 850; contents = 720.
875
default_priority = 800
877
v = NumberingVisitor(self.document)
878
self.document.walkabout(v)
879
self.document.reference_labels = v.reference_labels
880
self.document.callout_labels = v.callout_labels
882
class NumberReferences(Transform):
883
default_priority = 830
885
v = ReferenceVisitor(self.document, self.document.reference_labels,
886
self.document.callout_labels)
887
self.document.walkabout(v)
889
# Save reference info to a pickle file.
890
if OUTPUT_FORMAT == 'ref':
891
add_to_ref_file(reference_labels=self.document.reference_labels,
894
class NumberingVisitor(docutils.nodes.NodeVisitor):
896
A transforming visitor that adds figure numbers to all figures,
897
and converts any references to figures to use the text 'Figure #';
898
and adds example numbers to all examples, and converts any
899
references to examples to use the text 'Example #'.
901
LETTERS = 'abcdefghijklmnopqrstuvwxyz'
902
ROMAN = 'i ii iii iv v vi vii viii ix x'.split()
903
ROMAN += ['x%s' % r for r in ROMAN]
905
def __init__(self, document):
906
docutils.nodes.NodeVisitor.__init__(self, document)
907
self.reference_labels = {}
910
self.example_num = [0]
911
self.section_num = [0]
913
self.callout_labels = {} # name -> number
914
self.set_section_context = None
915
self.section_context = 'body' # preface, appendix, body
917
#////////////////////////////////////////////////////////////
919
#////////////////////////////////////////////////////////////
921
def visit_figure(self, node):
923
num = '%s.%s' % (self.format_section_num(1), self.figure_num)
924
for node_id in self.get_ids(node):
925
self.reference_labels[node_id] = '%s' % num
926
self.label_node(node, 'Figure %s' % num)
928
#////////////////////////////////////////////////////////////
930
#////////////////////////////////////////////////////////////
932
def visit_table(self, node):
933
if 'avm' in node['classes']: return
934
if 'gloss' in node['classes']: return
935
if 'rst-example' in node['classes']: return
936
if 'doctest-list' in node['classes']: return
938
num = '%s.%s' % (self.format_section_num(1), self.table_num)
939
for node_id in self.get_ids(node):
940
self.reference_labels[node_id] = '%s' % num
941
self.label_node(node, 'Table %s' % num)
943
#////////////////////////////////////////////////////////////
945
#////////////////////////////////////////////////////////////
947
def visit_pylisting(self, node):
948
self.listing_num += 1
949
num = '%s.%s' % (self.format_section_num(1), self.listing_num)
950
for node_id in self.get_ids(node):
951
self.reference_labels[node_id] = '%s' % num
952
pyfile = re.sub('\W', '_', node['name']) + PYLISTING_EXTENSION
953
self.label_node(node, 'Listing %s (%s)' % (num, pyfile),
954
PYLISTING_DIR + pyfile)
955
self.callout_labels.update(node['callouts'])
957
def visit_doctest_block(self, node):
958
if isinstance(node.parent, pylisting):
959
callouts = node['callouts'] = node.parent['callouts']
961
callouts = node['callouts'] = {}
963
pysrc = ''.join(('%s' % c) for c in node)
964
for callout_id in CALLOUT_RE.findall(pysrc):
965
callouts[callout_id] = len(callouts)+1
966
self.callout_labels.update(callouts)
968
#////////////////////////////////////////////////////////////
970
#////////////////////////////////////////////////////////////
971
max_section_depth = 3
972
no_section_numbers_in_preface = True
973
TOP_SECTION = 'chapter'
975
# [xx] I don't think this currently does anything..
976
def visit_document(self, node):
977
if (len(node)>0 and isinstance(node[0], docutils.nodes.title) and
978
isinstance(node[0].children[0], docutils.nodes.Text) and
979
re.match(r'(\d+(.\d+)*)\.?\s+', node[0].children[0].data)):
980
node['sectnum'] = node[0].children[0].data.split()[0]
981
for node_id in node.get('ids', []):
982
self.reference_labels[node_id] = '%s' % node['sectnum']
984
def visit_section(self, node):
987
# Check if we're entering a new context.
988
if len(self.section_num) == 1 and self.set_section_context:
989
self.start_new_context(node)
991
# Record the section's context in its title.
992
title['section_context'] = self.section_context
994
# Increment the section counter.
995
self.section_num[-1] += 1
997
# If a section number is given explicitly as part of the
998
# title, then it overrides our counter.
999
if isinstance(title.children[0], docutils.nodes.Text):
1000
m = re.match(r'(\d+(.\d+)*)\.?\s+', title.children[0].data)
1002
pieces = [int(n) for n in m.group(1).split('.')]
1003
if len(pieces) == len(self.section_num):
1004
self.section_num = pieces
1005
title.children[0].data = title.children[0].data[m.end():]
1007
warning('Explicit section number (%s) does not match '
1008
'current section depth' % m.group(1))
1009
self.prepend_raw_latex(node, r'\setcounter{%s}{%d}' %
1010
(self.TOP_SECTION, self.section_num[0]-1))
1012
# Record the reference pointer for this section; and add the
1013
# section number to the section title.
1014
node['sectnum'] = self.format_section_num()
1015
for node_id in node.get('ids', []):
1016
self.reference_labels[node_id] = '%s' % node['sectnum']
1017
if (len(self.section_num) <= self.max_section_depth and
1018
(OUTPUT_FORMAT != 'latex') and
1019
not (self.section_context == 'preface' and
1020
self.no_section_numbers_in_preface)):
1021
label = docutils.nodes.generated('', node['sectnum']+u'\u00a0'*3,
1022
classes=['sectnum'])
1023
title.insert(0, label)
1026
# Record the section number.
1027
self.section_num.append(0)
1029
# If this was a top-level section, then restart the figure,
1030
# table, and listing counters
1031
if len(self.section_num) == 2:
1034
self.listing_num = 0
1036
def start_new_context(self,node):
1037
# Set the 'section_context' var.
1038
self.section_context = self.set_section_context
1039
self.set_section_context = None
1041
# Update our counter.
1042
self.section_num[0] = 0
1044
# Update latex's counter.
1045
if self.section_context == 'preface': style = 'Roman'
1046
elif self.section_context == 'body': style = 'arabic'
1047
elif self.section_context == 'appendix': style = 'Alph'
1048
raw_latex = (('\n'+r'\setcounter{%s}{0}' + '\n' +
1049
r'\renewcommand \the%s{\%s{%s}}'+'\n') %
1050
(self.TOP_SECTION, self.TOP_SECTION, style, self.TOP_SECTION))
1051
if self.section_context == 'appendix':
1052
raw_latex += '\\appendix\n'
1053
self.prepend_raw_latex(node, raw_latex)
1055
def prepend_raw_latex(self, node, raw_latex):
1056
if isinstance(node, docutils.nodes.document):
1057
node.insert(0, docutils.nodes.raw('', raw_latex, format='latex'))
1059
node_index = node.parent.children.index(node)
1060
node.parent.insert(node_index, docutils.nodes.raw('', raw_latex,
1063
def depart_section(self, node):
1064
self.section_num.pop()
1066
def format_section_num(self, depth=None):
1067
pieces = [('%s' % p) for p in self.section_num]
1068
if self.section_context == 'body':
1069
pieces[0] = ('%s' % self.section_num[0])
1070
elif self.section_context == 'preface':
1071
pieces[0] = self.ROMAN[self.section_num[0]-1].upper()
1072
elif self.section_context == 'appendix':
1073
pieces[0] = self.LETTERS[self.section_num[0]-1].upper()
1075
assert 0, 'unexpected section context'
1077
return '.'.join(pieces)
1079
return '.'.join(pieces[:depth])
1082
def visit_section_context(self, node):
1083
assert node['context'] in ('body', 'preface', 'appendix')
1084
self.set_section_context = node['context']
1085
node.replace_self([])
1087
#////////////////////////////////////////////////////////////
1089
#////////////////////////////////////////////////////////////
1090
NESTED_EXAMPLES = True
1092
def visit_example(self, node):
1093
self.example_num[-1] += 1
1094
node['num'] = self.short_example_num()
1095
for node_id in self.get_ids(node):
1096
self.reference_labels[node_id] = self.format_example_num()
1097
self.example_num.append(0)
1099
def depart_example(self, node):
1100
if not self.NESTED_EXAMPLES:
1101
if self.example_num[-1] > 0:
1102
# If the example contains a list of subexamples, then
1103
# splice them in to our parent.
1104
node.replace_self(list(node))
1105
self.example_num.pop()
1107
def short_example_num(self):
1108
if len(self.example_num) == 1:
1109
return '(%s)' % self.example_num[0]
1110
if len(self.example_num) == 2:
1111
return '%s.' % self.LETTERS[self.example_num[1]-1]
1112
if len(self.example_num) == 3:
1113
return '%s.' % self.ROMAN[self.example_num[2]-1]
1115
return '%s.' % self.example_num[-1]
1117
def format_example_num(self):
1118
""" (1), (2); (1a), (1b); (1a.i), (1a.ii)"""
1119
ex_num = ('%s' % self.example_num[0])
1120
if len(self.example_num) > 1:
1121
ex_num += self.LETTERS[self.example_num[1]-1]
1122
if len(self.example_num) > 2:
1123
ex_num += '.%s' % self.ROMAN[self.example_num[2]-1]
1124
for n in self.example_num[3:]:
1126
return '(%s)' % ex_num
1128
#////////////////////////////////////////////////////////////
1130
#////////////////////////////////////////////////////////////
1132
def unknown_visit(self, node): pass
1133
def unknown_departure(self, node): pass
1135
def get_ids(self, node):
1136
node_index = node.parent.children.index(node)
1137
if node_index>0 and isinstance(node.parent[node_index-1],
1138
docutils.nodes.target):
1139
target = node.parent[node_index-1]
1140
if target.has_key('refid'):
1141
refid = target['refid']
1142
target['ids'] = [refid]
1145
elif target.has_key('ids'):
1146
return target['ids']
1148
warning('unable to find id for %s' % target)
1152
def label_node(self, node, label, refuri=None, cls='caption-label'):
1153
if not isinstance(node[-1], docutils.nodes.caption):
1154
node.append(docutils.nodes.caption())
1157
if OUTPUT_FORMAT == 'html':
1158
cap = docutils.nodes.inline('', label, classes=[cls])
1160
cap = docutils.nodes.reference('', '', cap, refuri=refuri,
1161
mimetype='text/x-python')
1162
caption.insert(0, cap)
1163
if len(caption) > 1:
1164
caption.insert(1, docutils.nodes.Text(': '))
1166
class ReferenceVisitor(docutils.nodes.NodeVisitor):
1167
def __init__(self, document, reference_labels, callout_labels):
1168
self.reference_labels = reference_labels
1169
self.callout_labels = callout_labels
1170
self.targets = set()
1171
docutils.nodes.NodeVisitor.__init__(self, document)
1172
def unknown_visit(self, node):
1173
if isinstance(node, docutils.nodes.Element):
1174
self.targets.update(node.get('names', []))
1175
self.targets.update(node.get('ids', []))
1176
def unknown_departure(self, node): pass
1178
# Don't mess with the table of contents.
1179
def visit_topic(self, node):
1180
if 'contents' in node.get('classes', ()):
1181
raise docutils.nodes.SkipNode
1183
def visit_reference(self, node):
1184
node_id = (node.get('refid') or
1185
self.document.nameids.get(node.get('refname')) or
1186
node.get('refname'))
1187
if node_id in self.reference_labels:
1188
label = self.reference_labels[node_id]
1190
node.append(docutils.nodes.Text(label))
1191
expand_reference_text(node)
1192
elif node_id in self.callout_labels:
1193
label = self.callout_labels[node_id]
1195
node.append(callout_marker(number=label, name='ref-%s' % node_id))
1196
expand_reference_text(node)
1197
# There's no explicitly encoded target element, so manually
1198
# resolve the reference:
1199
node['refid'] = node_id
1200
node.resolved = True
1202
_EXPAND_REF_RE = re.compile(r'(?is)^(.*)(%s)\s+$' % '|'.join(
1203
['figure', 'table', 'example', 'chapter', 'section', 'appendix',
1204
'sentence', 'tree', 'listing', 'program']))
1205
def expand_reference_text(node):
1206
"""If the reference is immediately preceeded by the word 'figure'
1207
or the word 'table' or 'example', then include that word in the
1208
link (rather than just the number)."""
1209
if node.get('expanded_ref'):
1210
assert 0, ('Already expanded!! %s' % node)
1211
node_index = node.parent.children.index(node)
1213
prev_node = node.parent.children[node_index-1]
1214
if (isinstance(prev_node, docutils.nodes.Text)):
1215
m = _EXPAND_REF_RE.match(prev_node.data)
1217
prev_node.data = m.group(1)
1218
link = node.children[0]
1219
link.data = '%s %s' % (m.group(2), link.data)
1220
node['expanded_ref'] = True
1222
######################################################################
1223
#{ Feature Structures (AVMs)
1224
######################################################################
1227
def __init__(self, ident):
1231
def assign(self, key, val):
1232
if key in self.keys: raise ValueError('duplicate key')
1233
self.keys.append(key)
1234
self.vals[key] = val
1237
for key in self.keys:
1238
val = self.vals[key]
1239
if isinstance(val, AVMPointer):
1240
vals.append('%s -> %s' % (key, val.ident))
1242
vals.append('%s = %s' % (key, val))
1243
s = '{%s}' % ', '.join(vals)
1244
if self.ident: s += '[%s]' % self.ident
1248
return '\\begin{avm}\n%s\\end{avm}\n' % self._as_latex()
1250
def _as_latex(self, indent=0):
1251
if self.ident: ident = '\\@%s ' % self.ident
1253
lines = ['%s %s & %s' % (indent*' ', key,
1254
self.vals[key]._as_latex(indent+1))
1255
for key in self.keys]
1256
return ident + '\\[\n' + ' \\\\\n'.join(lines) + '\\]\n'
1258
def _entry(self, val, cls):
1259
if isinstance(val, basestring):
1260
return docutils.nodes.entry('',
1261
docutils.nodes.paragraph('', val), classes=[cls])
1263
return docutils.nodes.entry('', val, classes=[cls])
1265
def _pointer(self, ident):
1266
return docutils.nodes.paragraph('', '',
1267
docutils.nodes.inline(ident, ident,
1268
classes=['avm-pointer']))
1271
return docutils.nodes.paragraph('', '[]',
1272
classes=['avm-empty'])
1275
for key in self.keys:
1276
val = self.vals[key]
1277
key_node = self._entry(key, 'avm-key')
1278
if isinstance(val, AVMPointer):
1279
eq_node = self._entry(u'\u2192', 'avm-eq') # right arrow
1280
val_node = self._entry(self._pointer(val.ident), 'avm-val')
1281
elif isinstance(val, AVM):
1282
eq_node = self._entry('=', 'avm-eq')
1283
val_node = self._entry(val.as_table(), 'avm-val')
1285
value = ('%s' % val.val).replace(' ', u'\u00a0') # =nbsp
1286
eq_node = self._entry('=', 'avm-eq')
1287
val_node = self._entry(value, 'avm-val')
1289
rows.append(docutils.nodes.row('', key_node, eq_node, val_node))
1291
# Add left/right bracket nodes:
1292
if len(self.keys)==1: vpos = 'topbot'
1293
elif key == self.keys[0]: vpos = 'top'
1294
elif key == self.keys[-1]: vpos = 'bot'
1296
rows[-1].insert(0, self._entry(u'\u00a0', 'avm-%sleft' % vpos))
1297
rows[-1].append(self._entry(u'\u00a0', 'avm-%sright' % vpos))
1300
if key == self.keys[0] and self.ident:
1301
rows[-1].append(self._entry(self._pointer(self.ident),
1304
rows[-1].append(self._entry(u'\u00a0', 'avm-ident'))
1306
colspecs = [docutils.nodes.colspec(colwidth=1) for i in range(6)]
1308
tbody = docutils.nodes.tbody('', *rows)
1309
tgroup = docutils.nodes.tgroup('', cols=3, *(colspecs+[tbody]))
1310
table = docutils.nodes.table('', tgroup, classes=['avm'])
1314
def __init__(self, ident, val):
1318
if self.ident: return '%s[%s]' % (self.val, self.ident)
1319
else: return '%r' % self.val
1320
def _as_latex(self, indent=0):
1321
return '%s' % self.val
1324
def __init__(self, ident):
1327
return '[%s]' % self.ident
1328
def _as_latex(self, indent=0):
1329
return '\\@{%s}' % self.ident
1331
def parse_avm(s, ident=None):
1332
lines = [l.rstrip() for l in s.split('\n') if l.strip()]
1333
if not lines: raise ValueError(0)
1334
lines.append('[%s]' % (' '*(len(lines[0])-2)))
1336
# Create our new AVM.
1339
w = len(lines[0]) # Line width
1340
avmval_pos = None # (left, right, top) for nested AVMs
1341
key = None # Key for nested AVMs
1342
ident = None # Identifier for nested AVMs
1344
NESTED = re.compile(r'\[\s+(\[.*\])\s*\]$')
1345
ASSIGN = re.compile(r'\[\s*(?P<KEY>[^\[=>]+?)\s*'
1347
r'(\((?P<ID>\d+)\))?\s*'
1348
r'((?P<VAL>.+?))\s*\]$')
1349
BLANK = re.compile(r'\[\s+\]$')
1351
for lineno, line in enumerate(lines):
1352
#debug('%s %s %s %r' % (lineno, key, avmval_pos, line))
1353
if line[0] != '[' or line[-1] != ']' or len(line) != w:
1354
raise ValueError(lineno)
1356
nested_m = NESTED.match(line)
1357
assign_m = ASSIGN.match(line)
1358
blank_m = BLANK.match(line)
1359
if not (nested_m or assign_m or blank_m):
1360
raise ValueError(lineno)
1362
if nested_m or (assign_m and assign_m.group('VAL').startswith('[')):
1363
left, right = line.index('[',1), line.rindex(']', 0, -1)+1
1364
if avmval_pos is None:
1365
avmval_pos = (left, right, lineno)
1366
elif avmval_pos[:2] != (left, right):
1367
raise ValueError(lineno)
1370
if assign_m.group('VAL').startswith('['):
1371
if key is not None: raise ValueError(lineno)
1372
if assign_m.group('EQ') != '=': raise ValueError(lineno)
1373
key = assign_m.group('KEY')
1374
ident = assign_m.group('ID')
1376
if assign_m.group('EQ') == '=':
1377
avm.assign(assign_m.group('KEY'),
1378
AVMValue(assign_m.group('ID'),
1379
assign_m.group('VAL')))
1381
if assign_m.group('VAL').strip(): raise ValueError(lineno)
1382
avm.assign(assign_m.group('KEY'),
1383
AVMPointer(assign_m.group('ID')))
1385
if blank_m and avmval_pos is not None:
1386
left, right, top = avmval_pos
1387
valstr = '\n'.join(l[left:right] for l in lines[top:lineno])
1388
avm.assign(key, parse_avm(valstr, ident))
1389
key = avmval_pos = None
1395
######################################################################
1396
#{ Doctest Indentation
1397
######################################################################
1399
class UnindentDoctests(Transform):
1401
In our source text, we have indented most of the doctest blocks,
1402
for two reasons: it makes copy/pasting with the doctest script
1403
easier; and it's more readable. But we don't *actually* want them
1404
to be included in block_quote environments when we output them.
1405
So this transform looks for any doctest_block's that are the only
1406
child of a block_quote, and eliminates the block_quote.
1408
default_priority = 1000
1410
self.document.walkabout(UnindentDoctestVisitor(self.document))
1412
class UnindentDoctestVisitor(docutils.nodes.NodeVisitor):
1413
def __init__(self, document):
1414
docutils.nodes.NodeVisitor.__init__(self, document)
1415
def unknown_visit(self, node): pass
1416
def unknown_departure(self, node): pass
1417
def visit_block_quote(self, node):
1418
if (len(node) == sum([1 for c in node if
1419
isinstance(c, docutils.nodes.doctest_block)])):
1420
node.replace_self(list(node))
1421
raise docutils.nodes.SkipNode()
1423
_OPTION_DIRECTIVE_RE = re.compile(
1424
r'(\n[ ]*\.\.\.[ ]*)?#\s*doctest:\s*([^\n\'"]*)$', re.MULTILINE)
1425
def strip_doctest_directives(text):
1426
return _OPTION_DIRECTIVE_RE.sub('', text)
1428
class pylisting(docutils.nodes.General, docutils.nodes.Element):
1430
Python source code listing.
1432
Children: doctest_block+ caption?
1434
######################################################################
1436
######################################################################
1437
from epydoc.docwriter.html_colorize import PythonSourceColorizer
1438
import epydoc.docwriter.html_colorize
1439
epydoc.docwriter.html_colorize .PYSRC_EXPANDTO_JAVASCRIPT = ''
1441
class CustomizedHTMLWriter(HTMLWriter):
1442
settings_defaults = HTMLWriter.settings_defaults.copy()
1443
settings_defaults.update({
1444
'stylesheet': CSS_STYLESHEET,
1445
'stylesheet_path': None,
1446
'output_encoding': 'unicode',
1447
'output_encoding_error_handler': 'xmlcharrefreplace',
1451
HTMLWriter.__init__(self)
1452
self.translator_class = CustomizedHTMLTranslator
1454
#def translate(self):
1455
# postprocess(self.document)
1456
# HTMLWriter.translate(self)
1458
class CustomizedHTMLTranslator(HTMLTranslator):
1459
def __init__(self, document):
1460
HTMLTranslator.__init__(self, document)
1461
self.head_prefix.append(COPY_CLIPBOARD_JS)
1463
def visit_pylisting(self, node):
1464
self._write_pylisting_file(node)
1465
self.body.append(self.CODEBOX_HEADER % ('pylisting', 'pylisting'))
1467
def depart_pylisting(self, node):
1468
self.body.append(self.CODEBOX_FOOTER)
1470
def visit_doctest_block(self, node):
1471
# Collect the text content of the doctest block.
1472
text = ''.join(('%s' % c) for c in node)
1473
text = textwrap.dedent(text)
1474
text = strip_doctest_directives(text)
1475
text = text.decode('latin1')
1477
# Colorize the contents of the doctest block.
1478
if hasattr(node, 'callouts'):
1479
callouts = node['callouts']
1482
colorizer = HTMLDoctestColorizer(self.encode, callouts)
1483
if node.get('is_codeblock'):
1484
pysrc = colorizer.colorize_codeblock(text)
1487
pysrc = colorizer.colorize_doctest(text)
1494
if node.get('is_codeblock'): typ = 'codeblock'
1495
else: typ = 'doctest'
1496
pysrc = self.CODEBOX_ROW % (typ, typ, pysrc)
1498
if not isinstance(node.parent, pylisting):
1499
self.body.append(self.CODEBOX_HEADER % ('doctest', 'doctest'))
1500
self.body.append(pysrc)
1501
self.body.append(self.CODEBOX_FOOTER)
1503
self.body.append(pysrc)
1505
raise docutils.nodes.SkipNode() # Content already processed
1507
CODEBOX_HEADER = ('<div class="%s">\n'
1508
'<table border="0" cellpadding="0" cellspacing="0" '
1509
'class="%s" width="95%%">\n')
1510
CODEBOX_FOOTER = '</table></div>\n'
1511
CODEBOX_ROW = textwrap.dedent('''\
1513
<table border="0" cellpadding="0" cellspacing="0" width="100%%">
1514
<tr><td width="1" class="copybar"
1515
onclick="javascript:copy_%s_to_clipboard(this.nextSibling);"
1517
<td class="pysrc">%s</td>
1518
</tr></table></td></tr>\n''')
1520
# For generated pylisting files:
1521
_PYLISTING_FILE_HEADER = "# Natural Language Toolkit: %s\n\n"
1523
def _write_pylisting_file(self, node):
1524
if not os.path.exists(PYLISTING_DIR):
1525
os.mkdir(PYLISTING_DIR)
1527
name = re.sub('\W', '_', node['name'])
1528
filename = os.path.join(PYLISTING_DIR, name+PYLISTING_EXTENSION)
1529
out = open(filename, 'w')
1530
out.write(self._PYLISTING_FILE_HEADER % name)
1532
if not isinstance(child, docutils.nodes.doctest_block):
1534
elif child['is_codeblock']:
1535
out.write(''.join(('%s' % c) for c in child)+'\n\n')
1536
elif INCLUDE_DOCTESTS_IN_PYLISTING_FILES:
1537
lines = ''.join(('%s' % c) for c in child).split('\n')
1538
in_doctest_block = False
1540
if line.startswith('>>> '):
1541
out.write(line[4:]+'\n')
1542
in_doctest_block = True
1543
elif line.startswith('... ') and in_doctest_block:
1544
out.write(line[4:]+'\n')
1546
if in_doctest_block:
1547
out.write('# Expect:\n')
1548
out.write('# ' + line+'\n')
1549
in_doctest_block = False
1551
out.write(line+'\n')
1552
in_doctest_block = False
1555
def visit_exercise(self, node):
1556
self.body.append('<exercise weight="1" src="')
1558
def depart_exercise(self, node):
1559
self.body.append('"/>')
1561
def visit_challenge(self, node):
1562
self.body.append('<exercise weight="0" src="')
1564
def depart_challenge(self, node):
1565
self.body.append('"/>')
1567
def visit_literal(self, node):
1568
"""Process text to prevent tokens from wrapping."""
1569
text = ''.join(('%s' % c) for c in node)
1570
text = text.decode('latin1')
1571
colorizer = HTMLDoctestColorizer(self.encode)
1572
pysrc = colorizer.colorize_inline(text)#.strip()
1573
#pysrc = colorize_doctestblock(text, self._markup_pysrc, True)
1574
self.body+= [self.starttag(node, 'tt', '', CLASS='doctest'),
1575
'<span class="pre">%s</span></tt>' % pysrc]
1576
raise docutils.nodes.SkipNode() # Content already processed
1578
def _markup_pysrc(self, s, tag):
1579
return '\n'.join('<span class="pysrc-%s">%s</span>' %
1580
(tag, self.encode(line))
1581
for line in s.split('\n'))
1583
def visit_example(self, node):
1585
'<p><table border="0" cellpadding="0" cellspacing="0" '
1586
'class="example">\n '
1587
'<tr valign="top"><td width="30" align="right">'
1588
'%s</td><td width="15"></td><td>' % node['num'])
1590
def depart_example(self, node):
1591
self.body.append('</td></tr></table></p>\n')
1593
def visit_idxterm(self, node):
1594
self.body.append('<span class="%s">' % ' '.join(node['classes']))
1595
if 'topic' in node['classes']: raise docutils.nodes.SkipChildren
1597
def depart_idxterm(self, node):
1598
self.body.append('</span>')
1600
def visit_index(self, node):
1601
self.body.append('<div class="index">\n<h1>Index</h1>\n')
1603
def depart_index(self, node):
1604
self.body.append('</div>\n')
1606
_seen_callout_markers = set()
1607
def visit_callout_marker(self, node):
1608
# Only add an id to a marker the first time we see it.
1609
add_id = (node['name'] not in self._seen_callout_markers)
1610
self._seen_callout_markers.add(node['name'])
1612
self.body.append('<span id="%s">' % node['name'])
1613
self.body.append(CALLOUT_IMG % (node['number'], node['number']))
1615
self.body.append('</span>')
1616
raise docutils.nodes.SkipNode() # Done with this node.
1618
def depart_field_name(self, node):
1619
# Don't add ":" in callout field lists.
1620
if 'callout' in node['classes']:
1621
self.body.append(self.context.pop())
1623
HTMLTranslator.depart_field_name(self, node)
1625
def _striphtml_len(self, s):
1626
return len(re.sub(r'&[^;]+;', 'x', re.sub(r'<[^<]+>', '', s)))
1628
def visit_caption(self, node):
1629
if isinstance(node.parent, pylisting):
1630
self.body.append('<tr><td class="caption">')
1631
HTMLTranslator.visit_caption(self, node)
1633
def depart_caption(self, node):
1634
if isinstance(node.parent, pylisting):
1635
self.body.append('</td></tr>')
1636
HTMLTranslator.depart_caption(self, node)
1638
def starttag(self, node, tagname, suffix='\n', empty=0, **attributes):
1639
if node.get('mimetype'):
1640
attributes['type'] = node.get('mimetype')
1641
return HTMLTranslator.starttag(self, node, tagname, suffix,
1642
empty, **attributes)
1644
######################################################################
1645
#{ Source Code Highlighting
1646
######################################################################
1648
# [xx] Note: requires the very latest svn version of epydoc!
1649
from epydoc.markup.doctest import DoctestColorizer
1651
class HTMLDoctestColorizer(DoctestColorizer):
1652
PREFIX = '<pre class="doctest">\n'
1654
def __init__(self, encode_func, callouts=None):
1655
self.encode = encode_func
1656
self.callouts = callouts
1657
def markup(self, s, tag):
1659
return self.encode(s)
1660
elif (tag == 'comment' and self.callouts is not None and
1661
CALLOUT_RE.match(s)):
1662
callout_id = CALLOUT_RE.match(s).group(1)
1663
callout_num = self.callouts[callout_id]
1664
img = CALLOUT_IMG % (callout_num, callout_num)
1665
return ('<a name="%s" /><a href="#ref-%s">%s</a>' %
1666
(callout_id, callout_id, img))
1668
return ('<span class="pysrc-%s">%s</span>' %
1669
(tag, self.encode(s)))
1671
######################################################################
1672
#{ Customized Reader (register new transforms)
1673
######################################################################
1675
class CustomizedReader(StandaloneReader):
1677
UnindentDoctests, # 1000
1679
def get_transforms(self):
1680
return StandaloneReader.get_transforms(self) + self._TRANSFORMS
1683
######################################################################
1685
######################################################################
1687
_OUTPUT_RE = re.compile(r'<div class="document">\s+(.*)\s+</div>\n</body>\n</html>',
1688
re.MULTILINE | re.DOTALL)
1692
CustomizedHTMLWriter.settings_defaults.update()
1693
output = docutils.core.publish_string(input,
1694
writer=CustomizedHTMLWriter(), reader=CustomizedReader())
1695
match = _OUTPUT_RE.search(output)
1697
return match.group(1)
1699
raise ValueError('Could not process exercise definition')
1701
except docutils.utils.SystemMessage, e:
1702
print 'Fatal error encountered!', e
added
removed
ivle/worksheet/rst.py
