~launchpad-pqm/launchpad/devel

#!/usr/bin/python
#
# Copyright (C) 2009 - Curtis Hovey <sinzui.is at verizon.net>
# This software is licensed under the GNU General Public License version 2.
#
# It comes from the Gedit Developer Plugins project (launchpad.net/gdp); see
# http://bazaar.launchpad.net/~sinzui/gdp/trunk/files/head%3A/plugins/gdp/ &
# http://bazaar.launchpad.net/%7Esinzui/gdp/trunk/annotate/head%3A/COPYING.

"""Reformat a doctest to Launchpad style."""

__metatype__ = type

import compiler
from difflib import unified_diff
from doctest import DocTestParser, Example
from optparse import OptionParser
import re
import sys
from textwrap import wrap

import pyflakes
from pyflakes.checker import Checker


class DoctestReviewer:
    """Check and reformat doctests."""
    rule_pattern = re.compile(r'([=~-])+[ ]*$')
    moin_pattern = re.compile(r'^(=+)[ ](.+)[ ](=+[ ]*)$')
    continuation_pattern = re.compile(r'^(\s*\.\.\.) (.+)$', re.M)

    SOURCE = 'source'
    WANT = 'want'
    NARRATIVE = 'narrative'

    def __init__(self, doctest, file_name):
        self.doctest = doctest
        self.file_name = file_name
        doctest = self._disambuguate_doctest(doctest)
        parser = DocTestParser()
        self.parts = parser.parse(doctest, file_name)
        self.blocks = []
        self.block = []
        self.block_method = self.preserve_block
        self.code_lines = []
        self.example = None
        self.last_bad_indent = 0
        self.has_printed_filename = False

    def _disambuguate_doctest(self, doctest):
        """Clarify continuations that the doctest parser hides."""
        return self.continuation_pattern.sub(r'\1    \2', doctest)

    def _print_message(self, message, lineno):
        """Print the error message with the lineno.

        :param message: The message to print.
        :param lineno: The line number the message pertains to.
        """
        if not self.has_printed_filename:
            print '%s:' % self.file_name
            self.has_printed_filename = True
        print '    % 4s: %s' % (lineno, message)

    def _is_formatted(self, text):
        """Return True if the text is pre-formatted, otherwise False.

        :param: text a string, or a list of strings.
        """
        if isinstance(text, list):
            text = text[0]
        return text.startswith(' ')

    def _walk(self, doctest_parts):
        """Walk the doctest parts; yield the line and kind.

        Yield the content of the line, and its kind (SOURCE, WANT, NARRATIVE).
        SOURCE and WANT lines are stripped of indentation, SOURCE is also
        stripped of the interpreter symbols.
        
        :param doctest_parts: The output of DocTestParser.parse.
        """
        for part in doctest_parts:
            if part == '':
                continue
            if isinstance(part, Example):
                self.example = part
                for line in part.source.splitlines():
                    kind = DoctestReviewer.SOURCE
                    yield line, kind
                for line in part.want.splitlines():
                    kind = DoctestReviewer.WANT
                    yield line, kind
            else:
                self.example = None
                kind = DoctestReviewer.NARRATIVE
                for line in part.splitlines():
                    yield line, kind

    def _apply(self, line_methods):
        """Call each line_method for each line in the doctest.

        :param line_methods: a list of methods that accept lineno, line,
            and kind as arguments. Each method must return the line for
            the next method to process.
        """
        self.blocks = []
        self.block = []
        lineno = 0
        previous_kind = DoctestReviewer.NARRATIVE
        for line, kind in self._walk(self.parts):
            lineno += 1
            self._append_source(kind, line)
            if kind != previous_kind and kind != DoctestReviewer.WANT:
                # The WANT block must adjoin the preceding SOURCE block.
                self._store_block(previous_kind)
            for method in line_methods:
                line = method(lineno, line, kind, previous_kind)
                if line is None:
                    break
            if not line:
                continue
            self.block.append(line)
            previous_kind = kind
        # Capture the last block and a blank line.
        self.block.append('\n')
        self._store_block(previous_kind)

    def _append_source(self, kind, line):
        """Update the list of source code lines seen."""
        if kind == self.SOURCE:
            self.code_lines.append(line)
        else:
            self.code_lines.append('')

    def _store_block(self, kind):
        """Append the block to blocks, re-wrap unformatted narrative.

        :param kind: The block's kind (SOURCE, WANT, NARRATIVE)
        """
        if len(self.block) == 0:
            return
        block = self.block_method(kind, self.block, self.blocks)
        self.blocks.append('\n'.join(block))
        self.block = []

    def check(self):
        """Check the doctest for style and code issues.

        1. Check line lengths.
        2. Check that headings are not in Moin format.
        3. Check indentation.
        4. Check trailing whitespace.
        """
        self.code_lines = []
        line_checkers = [
            self.check_length,
            self.check_heading,
            self.check_indentation,
            self.check_trailing_whitespace,]
        self._apply(line_checkers)
        code = '\n'.join(self.code_lines)
        self.check_source_code(code)

    def format(self):
        """Reformat doctest.

        1. Tests are reindented to 4 spaces.
        2. Simple narrative is rewrapped to 78 character width.
        3. Formatted (indented) narrative is preserved.
        4. Moin headings are converted to RSR =, == , and === levels.
        5. There is one blank line between blocks,
        6. Except for headers which have two leading blank lines.
        7. All trailing whitespace is removed.

        SOURCE and WANT long lines are not fixed--this is a human operation.
        """
        line_checkers = [
            self.fix_trailing_whitespace,
            self.fix_indentation,
            self.fix_heading,
            self.fix_narrative_paragraph,]
        self.block_method = self.format_block
        self._apply(line_checkers)
        self.block_method = self.preserve_block
        return '\n\n'.join(self.blocks)

    def preserve_block(self, kind, block, blocks):
        """Do nothing to the block.

        :param kind: The block's kind (SOURCE, WANT, NARRATIVE)
        :param block: The list of lines that should remain together.
        :param blocks: The list of all collected blocks.
        """
        return block

    def format_block(self, kind, block, blocks):
        """Format paragraph blocks.

        :param kind: The block's kind (SOURCE, WANT, NARRATIVE)
        :param block: The list of lines that should remain together.
        :param blocks: The list of all collected blocks.
        """
        if kind != DoctestReviewer.NARRATIVE or self._is_formatted(block):
            return block
        try:
            rules = ('===', '---', '...')
            last_line = block[-1]
            is_heading = last_line[0:3] in rules and last_line[-3:] in rules
        except IndexError:
            is_heading = False
        if len(blocks) != 0 and is_heading:
            # Headings should have an extra leading blank line.
            block.insert(0, '')
        elif is_heading:
            # Do nothing. This is the first heading in the file.
            pass
        else:
            long_line = ' '.join(block).strip()
            block = wrap(long_line, 72)
        return block

    def is_comment(self, line):
        """Return True if the line is a comment."""
        comment_pattern = re.compile(r'\s*#')
        return comment_pattern.match(line) is not None

    def check_length(self, lineno, line, kind, previous_kind):
        """Check the length of the line.

        Each kind of line has a maximum length:

        * NARRATIVE: 78 characters.
        * SOURCE: 70 characters (discounting indentation and interpreter).
        * WANT: 74 characters (discounting indentation).
        """

        length = len(line)
        if kind == DoctestReviewer.NARRATIVE and self.is_comment(line):
            # comments follow WANT rules because they are in code.
            kind = DoctestReviewer.WANT
            line = line.lstrip()
        if kind == DoctestReviewer.NARRATIVE and length > 78:
            self._print_message('%s exceeds 78 characters.' % kind, lineno)
        elif kind == DoctestReviewer.WANT and length > 74:
            self._print_message('%s exceeds 78 characters.' % kind, lineno)
        elif kind == DoctestReviewer.SOURCE and length > 70:
            self._print_message('%s exceeds 78 characters.' % kind, lineno)
        else:
            # This line has a good length.
            pass
        return line

    def check_indentation(self, lineno, line, kind, previous_kind):
        """Check the indentation of the SOURCE or WANT line."""
        if kind == DoctestReviewer.NARRATIVE:
            return line
        if self.example.indent != 4:
            if self.last_bad_indent != lineno - 1:
                self._print_message('%s has bad indentation.' % kind, lineno)
            self.last_bad_indent = lineno
        return line

    def check_trailing_whitespace(self, lineno, line, kind, previous_kind):
        """Check for the presence of trailing whitespace in the line."""
        if line.endswith(' '):
            self._print_message('%s has trailing whitespace.' % kind, lineno)
        return line

    def check_heading(self, lineno, line, kind, previous_kind):
        """Check for narrative lines that use moin headers instead of RST."""
        if kind != DoctestReviewer.NARRATIVE:
            return line
        moin = self.moin_pattern.match(line)
        if moin is not None:
            self._print_message('%s uses a moin header.' % kind, lineno - 1)
        return line

    def check_source_code(self, code):
        """Check for source code problems in the doctest using pyflakes.

        The most common problem found are unused imports. `UndefinedName`
        errors are suppressed because the test setup is not known.
        """
        if code == '':
            return
        try:
            tree = compiler.parse(code)
        except (SyntaxError, IndentationError), exc:
            (lineno, offset_, line) = exc[1][1:]
            if line.endswith("\n"):
                line = line[:-1]
            self._print_message(
                'Could not compile:\n          %s' % line, lineno - 1)
        else:
            warnings = Checker(tree)
            for warning in warnings.messages:
                if isinstance(warning, pyflakes.messages.UndefinedName):
                    continue
                dummy, lineno, message = str(warning).split(':')
                self._print_message(message.strip(), lineno)

    def fix_trailing_whitespace(self, lineno, line, kind, previous_kind):
        """Return the line striped of trailing whitespace."""
        return line.rstrip()

    def fix_indentation(self, lineno, line, kind, previous_kind):
        """set the indentation to 4-spaces."""
        if kind == DoctestReviewer.NARRATIVE:
            return line
        elif kind == DoctestReviewer.WANT:
            return '    %s' % line
        else:
            if line.startswith(' '):
                # This is a continuation of DoctestReviewer.SOURCE.
                return '    ... %s' % line
            else:
                # This is a start of DoctestReviewer.SOURCE.
                return '    >>> %s' % line

    def fix_heading(self, lineno, line, kind, previous_kind):
        """Switch Moin headings to RST headings."""
        if kind != DoctestReviewer.NARRATIVE:
            return line
        moin = self.moin_pattern.match(line)
        if moin is None:
            return line
        heading_level = len(moin.group(1))
        heading = moin.group(2)
        rule_length = len(heading)
        if heading_level == 1:
            rule = '=' * rule_length
        elif heading_level == 2:
            rule = '-' * rule_length
        else:
            rule = '.' * rule_length
        # Force the heading on to the block of lines.
        self.block.append(heading)
        return rule

    def fix_narrative_paragraph(self, lineno, line, kind, previous_kind):
        """Break narrative into paragraphs."""
        if kind != DoctestReviewer.NARRATIVE or len(self.block) == 0:
            return line
        if line == '':
            # This is the start of a new paragraph in the narrative.
            self._store_block(previous_kind)
        if self._is_formatted(line) and not self._is_formatted(self.block):
            # This line starts a pre-formatted paragraph.
            self._store_block(previous_kind)
        return line


def get_option_parser():
    """Return the option parser for this program."""
    usage = "usage: %prog [options] doctest.txt"
    parser = OptionParser(usage=usage)
    parser.add_option(
        "-f", "--format", dest="is_format", action="store_true",
        help="Reformat the doctest.")
    parser.add_option(
        "-i", "--interactive", dest="is_interactive",  action="store_true",
        help="Approve each change.")
    parser.set_defaults(
        is_format=False,
        is_interactive=False)
    return parser


def main(argv=None):
    """Run the operations requested from the command line."""
    if argv is None:
        argv = sys.argv
    parser = get_option_parser()
    (options, args) = parser.parse_args(args=argv[1:])
    if len(args) == 0:
        parser.error("A doctest must be specified.")

    for file_name in args:
        try:
            doctest_file = open(file_name)
            old_doctest = doctest_file.read()
        finally:
            doctest_file.close()
        reviewer = DoctestReviewer(old_doctest, file_name)

        if not options.is_format:
            reviewer.check()
            continue

        new_doctest = reviewer.format()
        if new_doctest != old_doctest:
            if options.is_interactive:
                diff = unified_diff(
                    old_doctest.splitlines(), new_doctest.splitlines())
                print '\n'.join(diff)
                print '\n'
                do_save = raw_input(
                    'Do you wish to save the changes? S(ave) or C(ancel)?')
            else:
                do_save = 'S'

            if do_save.upper() == 'S':
                try:
                    doctest_file = open(file_name, 'w')
                    doctest_file.write(new_doctest)
                finally:
                    doctest_file.close()
            reviewer = DoctestReviewer(new_doctest, file_name)
            reviewer.check()


if __name__ == '__main__':
    sys.exit(main())