~launchpad-pqm/launchpad/devel

"""Beautiful Soup

Elixir and Tonic

"The Screen-Scraper's Friend"

v2.1.0

http://www.crummy.com/software/BeautifulSoup/

Beautiful Soup parses arbitrarily invalid XML- or HTML-like substance

into a tree representation. It provides methods and Pythonic idioms

that make it easy to search and modify the tree.

A well-formed XML/HTML document will yield a well-formed data

structure. An ill-formed XML/HTML document will yield a

correspondingly ill-formed data structure. If your document is only

locally well-formed, you can use this library to find and process the

well-formed part of it. The BeautifulSoup class has heuristics for

obtaining a sensible parse tree in the face of common HTML errors.

Beautiful Soup has no external dependencies. It works with Python 2.2

and up.

Beautiful Soup defines classes for four different parsing strategies:

 * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific

   language that kind of looks like XML.

 * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid

   or invalid.

 * ICantBelieveItsBeautifulSoup, for parsing valid but bizarre HTML

   that trips up BeautifulSoup.

 * BeautifulSOAP, for making it easier to parse XML documents that use

   lots of subelements containing a single string, where you'd prefer

   they put that string into an attribute (such as SOAP messages).

You can subclass BeautifulStoneSoup or BeautifulSoup to create a

parsing strategy specific to an XML schema or a particular bizarre

HTML document. Typically your subclass would just override

SELF_CLOSING_TAGS and/or NESTABLE_TAGS.

"""

from __future__ import generators

__author__ = "Leonard Richardson (leonardr@segfault.org)"

__version__ = "2.1.0"

__copyright__ = "Copyright (c) 2004-2005 Leonard Richardson"

__license__ = "PSF"

from sgmllib import SGMLParser, SGMLParseError

import types

import re

import sgmllib

#This code makes Beautiful Soup able to parse XML with namespaces

sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*')

class NullType(object):

    """Similar to NoneType with a corresponding singleton instance

    'Null' that, unlike None, accepts any message and returns itself.

    Examples:

    >>> Null("send", "a", "message")("and one more",

    ...      "and what you get still") is Null

    True

    """

    def __new__(cls):                    return Null

    def __call__(self, *args, **kwargs): return Null

##    def __getstate__(self, *args):       return Null

    def __getattr__(self, attr):         return Null

    def __getitem__(self, item):         return Null

    def __setattr__(self, attr, value):  pass

    def __setitem__(self, item, value):  pass

    def __len__(self):                   return 0

    # FIXME: is this a python bug? otherwise ``for x in Null: pass``

    #        never terminates...

    def __iter__(self):                  return iter([])

    def __contains__(self, item):        return False

    def __repr__(self):                  return "Null"

Null = object.__new__(NullType)

class PageElement:

    """Contains the navigational information for some part of the page

    (either a tag or a piece of text)"""

    def setup(self, parent=Null, previous=Null):

        """Sets up the initial relations between this element and

        other elements."""

        self.parent = parent

        self.previous = previous

        self.next = Null

        self.previousSibling = Null

        self.nextSibling = Null

        if self.parent and self.parent.contents:

            self.previousSibling = self.parent.contents[-1]

            self.previousSibling.nextSibling = self

    def findNext(self, name=None, attrs={}, text=None):

        """Returns the first item that matches the given criteria and

        appears after this Tag in the document."""

        return self._first(self.fetchNext, name, attrs, text)

    firstNext = findNext

    def fetchNext(self, name=None, attrs={}, text=None, limit=None):

        """Returns all items that match the given criteria and appear

        before after Tag in the document."""

        return self._fetch(name, attrs, text, limit, self.nextGenerator)

    def findNextSibling(self, name=None, attrs={}, text=None):

        """Returns the closest sibling to this Tag that matches the

        given criteria and appears after this Tag in the document."""

        return self._first(self.fetchNextSiblings, name, attrs, text)

    firstNextSibling = findNextSibling

    def fetchNextSiblings(self, name=None, attrs={}, text=None, limit=None):

        """Returns the siblings of this Tag that match the given

        criteria and appear after this Tag in the document."""

        return self._fetch(name, attrs, text, limit, self.nextSiblingGenerator)

    def findPrevious(self, name=None, attrs={}, text=None):

        """Returns the first item that matches the given criteria and

        appears before this Tag in the document."""

        return self._first(self.fetchPrevious, name, attrs, text)

    def fetchPrevious(self, name=None, attrs={}, text=None, limit=None):

        """Returns all items that match the given criteria and appear

        before this Tag in the document."""

        return self._fetch(name, attrs, text, limit, self.previousGenerator)

    firstPrevious = findPrevious

    def findPreviousSibling(self, name=None, attrs={}, text=None):

        """Returns the closest sibling to this Tag that matches the

        given criteria and appears before this Tag in the document."""

        return self._first(self.fetchPreviousSiblings, name, attrs, text)

    firstPreviousSibling = findPreviousSibling

    def fetchPreviousSiblings(self, name=None, attrs={}, text=None,

                              limit=None):

        """Returns the siblings of this Tag that match the given

        criteria and appear before this Tag in the document."""

        return self._fetch(name, attrs, text, limit,

                           self.previousSiblingGenerator)

    def findParent(self, name=None, attrs={}):

        """Returns the closest parent of this Tag that matches the given

        criteria."""

        r = Null

        l = self.fetchParents(name, attrs, 1)

        if l:

            r = l[0]

        return r

    firstParent = findParent

    def fetchParents(self, name=None, attrs={}, limit=None):

        """Returns the parents of this Tag that match the given

        criteria."""

        return self._fetch(name, attrs, None, limit, self.parentGenerator)

    #These methods do the real heavy lifting.

    def _first(self, method, name, attrs, text):

        r = Null

        l = method(name, attrs, text, 1)

        if l:

            r = l[0]

        return r

    def _fetch(self, name, attrs, text, limit, generator):

        "Iterates over a generator looking for things that match."

        if not hasattr(attrs, 'items'):

            attrs = {'class' : attrs}

        results = []

        g = generator()

        while True:

            try:

                i = g.next()

            except StopIteration:

                break

            found = None

            if isinstance(i, Tag):

                if not text:

                    if not name or self._matches(i, name):

                        match = True

                        for attr, matchAgainst in attrs.items():

                            check = i.get(attr)

                            if not self._matches(check, matchAgainst):

                                match = False

                                break

                        if match:

                            found = i

            elif text:

                if self._matches(i, text):

                    found = i                    

            if found:

                results.append(found)

                if limit and len(results) >= limit:

                    break

        return results

    #Generators that can be used to navigate starting from both

    #NavigableTexts and Tags.                

    def nextGenerator(self):

        i = self

        while i:

            i = i.next

            yield i

    def nextSiblingGenerator(self):

        i = self

        while i:

            i = i.nextSibling

            yield i

    def previousGenerator(self):

        i = self

        while i:

            i = i.previous

            yield i

    def previousSiblingGenerator(self):

        i = self

        while i:

            i = i.previousSibling

            yield i

    def parentGenerator(self):

        i = self

        while i:

            i = i.parent

            yield i

    def _matches(self, chunk, howToMatch):

        #print 'looking for %s in %s' % (howToMatch, chunk)

        #

        # If given a list of items, return true if the list contains a

        # text element that matches.

        if isList(chunk) and not isinstance(chunk, Tag):

            for tag in chunk:

                if isinstance(tag, NavigableText) and self._matches(tag, howToMatch):

                    return True

            return False

        if callable(howToMatch):

            return howToMatch(chunk)

        if isinstance(chunk, Tag):

            #Custom match methods take the tag as an argument, but all other

            #ways of matching match the tag name as a string

            chunk = chunk.name

        #Now we know that chunk is a string

        if not type(chunk) in types.StringTypes:

            chunk = str(chunk)

        if hasattr(howToMatch, 'match'):

            # It's a regexp object.

            return howToMatch.search(chunk)

        if isList(howToMatch):

            return chunk in howToMatch

        if hasattr(howToMatch, 'items'):

            return howToMatch.has_key(chunk)

        #It's just a string

        return str(howToMatch) == chunk

class NavigableText(PageElement):

    def __getattr__(self, attr):

        "For backwards compatibility, text.string gives you text"

        if attr == 'string':

            return self

        else:

            raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr)

class NavigableString(str, NavigableText):

    pass

class NavigableUnicodeString(unicode, NavigableText):

    pass

class Tag(PageElement):

    """Represents a found HTML tag with its attributes and contents."""

    def __init__(self, name, attrs=None, parent=Null, previous=Null):

        "Basic constructor."

        self.name = name

        if attrs == None:

            attrs = []

        self.attrs = attrs

        self.contents = []

        self.setup(parent, previous)

        self.hidden = False

    def get(self, key, default=None):

        """Returns the value of the 'key' attribute for the tag, or

        the value given for 'default' if it doesn't have that

        attribute."""

        return self._getAttrMap().get(key, default)    

    def __getitem__(self, key):

        """tag[key] returns the value of the 'key' attribute for the tag,

        and throws an exception if it's not there."""

        return self._getAttrMap()[key]

    def __iter__(self):

        "Iterating over a tag iterates over its contents."

        return iter(self.contents)

    def __len__(self):

        "The length of a tag is the length of its list of contents."

        return len(self.contents)

    def __contains__(self, x):

        return x in self.contents

    def __nonzero__(self):

        "A tag is non-None even if it has no contents."

        return True

    def __setitem__(self, key, value):        

        """Setting tag[key] sets the value of the 'key' attribute for the

        tag."""

        self._getAttrMap()

        self.attrMap[key] = value

        found = False

        for i in range(0, len(self.attrs)):

            if self.attrs[i][0] == key:

                self.attrs[i] = (key, value)

                found = True

        if not found:

            self.attrs.append((key, value))

        self._getAttrMap()[key] = value

    def __delitem__(self, key):

        "Deleting tag[key] deletes all 'key' attributes for the tag."

        for item in self.attrs:

            if item[0] == key:

                self.attrs.remove(item)

                #We don't break because bad HTML can define the same

                #attribute multiple times.

            self._getAttrMap()

            if self.attrMap.has_key(key):

                del self.attrMap[key]

    def __call__(self, *args, **kwargs):

        """Calling a tag like a function is the same as calling its

        fetch() method. Eg. tag('a') returns a list of all the A tags

        found within this tag."""

        return apply(self.fetch, args, kwargs)

    def __getattr__(self, tag):

        if len(tag) > 3 and tag.rfind('Tag') == len(tag)-3:

            return self.first(tag[:-3])

        elif tag.find('__') != 0:

            return self.first(tag)

    def __eq__(self, other):

        """Returns true iff this tag has the same name, the same attributes,

        and the same contents (recursively) as the given tag.

        NOTE: right now this will return false if two tags have the

        same attributes in a different order. Should this be fixed?"""

        if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other):

            return False

        for i in range(0, len(self.contents)):

            if self.contents[i] != other.contents[i]:

                return False

        return True

    def __ne__(self, other):

        """Returns true iff this tag is not identical to the other tag,

        as defined in __eq__."""

        return not self == other

    def __repr__(self):

        """Renders this tag as a string."""

        return str(self)

    def __unicode__(self):

        return self.__str__(1)

    def __str__(self, needUnicode=None, showStructureIndent=None):

        """Returns a string or Unicode representation of this tag and

        its contents.

        NOTE: since Python's HTML parser consumes whitespace, this

        method is not certain to reproduce the whitespace present in

        the original string."""

        attrs = []

        if self.attrs:

            for key, val in self.attrs:

                attrs.append('%s="%s"' % (key, val))

        close = ''

        closeTag = ''

        if self.isSelfClosing():

            close = ' /'

        else:

            closeTag = '</%s>' % self.name

        indentIncrement = None        

        if showStructureIndent != None:

            indentIncrement = showStructureIndent

            if not self.hidden:

                indentIncrement += 1

        contents = self.renderContents(indentIncrement, needUnicode=needUnicode)        

        if showStructureIndent:

            space = '\n%s' % (' ' * showStructureIndent)

        if self.hidden:

            s = contents

        else:

            s = []

            attributeString = ''

            if attrs:

                attributeString = ' ' + ' '.join(attrs)            

            if showStructureIndent:

                s.append(space)

            s.append('<%s%s%s>' % (self.name, attributeString, close))

            s.append(contents)

            if closeTag and showStructureIndent != None:

                s.append(space)

            s.append(closeTag)

            s = ''.join(s)

        isUnicode = type(s) == types.UnicodeType

        if needUnicode and not isUnicode:

            s = unicode(s)

        elif isUnicode and needUnicode==False:

            s = str(s)

        return s

    def prettify(self, needUnicode=None):

        return self.__str__(needUnicode, showStructureIndent=True)

    def renderContents(self, showStructureIndent=None, needUnicode=None):

        """Renders the contents of this tag as a (possibly Unicode) 

        string."""

        s=[]

        for c in self:

            text = None

            if isinstance(c, NavigableUnicodeString) or type(c) == types.UnicodeType:

                text = unicode(c)

            elif isinstance(c, Tag):

                s.append(c.__str__(needUnicode, showStructureIndent))

            elif needUnicode:

                text = unicode(c)

            else:

                text = str(c)

            if text:

                if showStructureIndent != None:

                    if text[-1] == '\n':

                        text = text[:-1]

                s.append(text)

        return ''.join(s)    

    #Soup methods

    def firstText(self, text, recursive=True):

        """Convenience method to retrieve the first piece of text matching the

        given criteria. 'text' can be a string, a regular expression object,

        a callable that takes a string and returns whether or not the

        string 'matches', etc."""

        return self.first(recursive=recursive, text=text)

    def fetchText(self, text, recursive=True, limit=None):

        """Convenience method to retrieve all pieces of text matching the

        given criteria. 'text' can be a string, a regular expression object,

        a callable that takes a string and returns whether or not the

        string 'matches', etc."""

        return self.fetch(recursive=recursive, text=text, limit=limit)

    def first(self, name=None, attrs={}, recursive=True, text=None):

        """Return only the first child of this

        Tag matching the given criteria."""

        r = Null

        l = self.fetch(name, attrs, recursive, text, 1)

        if l:

            r = l[0]

        return r

    findChild = first

    def fetch(self, name=None, attrs={}, recursive=True, text=None,

              limit=None):

        """Extracts a list of Tag objects that match the given

        criteria.  You can specify the name of the Tag and any

        attributes you want the Tag to have.

        The value of a key-value pair in the 'attrs' map can be a

        string, a list of strings, a regular expression object, or a

        callable that takes a string and returns whether or not the

        string matches for some custom definition of 'matches'. The

        same is true of the tag name."""

        generator = self.recursiveChildGenerator

        if not recursive:

            generator = self.childGenerator

        return self._fetch(name, attrs, text, limit, generator)

    fetchChildren = fetch

    #Utility methods

    def isSelfClosing(self):

        """Returns true iff this is a self-closing tag as defined in the HTML

        standard.

        TODO: This is specific to BeautifulSoup and its subclasses, but it's

        used by __str__"""

        return self.name in BeautifulSoup.SELF_CLOSING_TAGS

    def append(self, tag):

        """Appends the given tag to the contents of this tag."""

        self.contents.append(tag)

    #Private methods

    def _getAttrMap(self):

        """Initializes a map representation of this tag's attributes,

        if not already initialized."""

        if not getattr(self, 'attrMap'):

            self.attrMap = {}

            for (key, value) in self.attrs:

                self.attrMap[key] = value 

        return self.attrMap

    #Generator methods

    def childGenerator(self):

        for i in range(0, len(self.contents)):

            yield self.contents[i]

        raise StopIteration

    def recursiveChildGenerator(self):

        stack = [(self, 0)]

        while stack:

            tag, start = stack.pop()

            if isinstance(tag, Tag):            

                for i in range(start, len(tag.contents)):

                    a = tag.contents[i]

                    yield a

                    if isinstance(a, Tag) and tag.contents:

                        if i < len(tag.contents) - 1:

                            stack.append((tag, i+1))

                        stack.append((a, 0))

                        break

        raise StopIteration

def isList(l):

    """Convenience method that works with all 2.x versions of Python

    to determine whether or not something is listlike."""

    return hasattr(l, '__iter__') \

           or (type(l) in (types.ListType, types.TupleType))

def buildTagMap(default, *args):

    """Turns a list of maps, lists, or scalars into a single map.

    Used to build the SELF_CLOSING_TAGS and NESTABLE_TAGS maps out

    of lists and partial maps."""

    built = {}

    for portion in args:

        if hasattr(portion, 'items'):

            #It's a map. Merge it.

            for k,v in portion.items():

                built[k] = v

        elif isList(portion):

            #It's a list. Map each item to the default.

            for k in portion:

                built[k] = default

        else:

            #It's a scalar. Map it to the default.

            built[portion] = default

    return built

class BeautifulStoneSoup(Tag, SGMLParser):

    """This class contains the basic parser and fetch code. It defines

    a parser that knows nothing about tag behavior except for the

    following:

      You can't close a tag without closing all the tags it encloses.

      That is, "<foo><bar></foo>" actually means

      "<foo><bar></bar></foo>".

    [Another possible explanation is "<foo><bar /></foo>", but since

    this class defines no SELF_CLOSING_TAGS, it will never use that

    explanation.]

    This class is useful for parsing XML or made-up markup languages,

    or when BeautifulSoup makes an assumption counter to what you were

    expecting."""

    SELF_CLOSING_TAGS = {}

    NESTABLE_TAGS = {}

    QUOTE_TAGS = {}

    #As a public service we will by default silently replace MS smart quotes

    #and similar characters with their HTML or ASCII equivalents.

    MS_CHARS = { '\x80' : 'euro',

                 '\x81' : ' ',

                 '\x82' : 'sbquo',

                 '\x83' : 'fnof',

                 '\x84' : 'bdquo',

                 '\x85' : 'hellip',

                 '\x86' : 'dagger',

                 '\x87' : 'Dagger',

                 '\x88' : 'caret',

                 '\x89' : '%',

                 '\x8A' : 'Scaron',

                 '\x8B' : 'lt;',

                 '\x8C' : 'OElig',

                 '\x8D' : '?',

                 '\x8E' : 'Z',

                 '\x8F' : '?',

                 '\x90' : '?',

                 '\x91' : 'lsquo',

                 '\x92' : 'rsquo',

                 '\x93' : 'ldquo',

                 '\x94' : 'rdquo',

                 '\x95' : 'bull',

                 '\x96' : 'ndash',

                 '\x97' : 'mdash',

                 '\x98' : 'tilde',

                 '\x99' : 'trade',

                 '\x9a' : 'scaron',

                 '\x9b' : 'gt',

                 '\x9c' : 'oelig',

                 '\x9d' : '?',

                 '\x9e' : 'z',

                 '\x9f' : 'Yuml',}

    PARSER_MASSAGE = [(re.compile('(<[^<>]*)/>'),

                       lambda(x):x.group(1) + ' />'),

                      (re.compile('<!\s+([^<>]*)>'),

                       lambda(x):'<!' + x.group(1) + '>'),

                      (re.compile("([\x80-\x9f])", re.M),

                       lambda(x): '&' + BeautifulStoneSoup.MS_CHARS.get(x.group(1)) + ';')

                      ]

    ROOT_TAG_NAME = '[document]'

    def __init__(self, text=None, avoidParserProblems=True,

                 initialTextIsEverything=True):

        """Initialize this as the 'root tag' and feed in any text to

        the parser.

        NOTE about avoidParserProblems: sgmllib will process most bad

        HTML, and BeautifulSoup has tricks for dealing with some HTML

        that kills sgmllib, but Beautiful Soup can nonetheless choke

        or lose data if your data uses self-closing tags or

        declarations incorrectly. By default, Beautiful Soup sanitizes

        its input to avoid the vast majority of these problems. The

        problems are relatively rare, even in bad HTML, so feel free

        to pass in False to avoidParserProblems if they don't apply to

        you, and you'll get better performance. The only reason I have

        this turned on by default is so I don't get so many tech

        support questions.

        The two most common instances of invalid HTML that will choke

        sgmllib are fixed by the default parser massage techniques:

         <br/> (No space between name of closing tag and tag close)

         <! --Comment--> (Extraneous whitespace in declaration)

        You can pass in a custom list of (RE object, replace method)

        tuples to get Beautiful Soup to scrub your input the way you

        want."""

        Tag.__init__(self, self.ROOT_TAG_NAME)

        if avoidParserProblems \

           and not isList(avoidParserProblems):

            avoidParserProblems = self.PARSER_MASSAGE            

        self.avoidParserProblems = avoidParserProblems

        SGMLParser.__init__(self)

        self.quoteStack = []

        self.hidden = 1

        self.reset()

        if hasattr(text, 'read'):

            #It's a file-type object.

            text = text.read()

        if text:

            self.feed(text)

        if initialTextIsEverything:

            self.done()

    def __getattr__(self, methodName):

        """This method routes method call requests to either the SGMLParser

        superclass or the Tag superclass, depending on the method name."""

        if methodName.find('start_') == 0 or methodName.find('end_') == 0 \

               or methodName.find('do_') == 0:

            return SGMLParser.__getattr__(self, methodName)

        elif methodName.find('__') != 0:

            return Tag.__getattr__(self, methodName)

        else:

            raise AttributeError

    def feed(self, text):

        if self.avoidParserProblems:

            for fix, m in self.avoidParserProblems:

                text = fix.sub(m, text)

        SGMLParser.feed(self, text)

        self.endData()

    def done(self):

        """Called when you're done parsing, so that the unclosed tags can be

        correctly processed."""

        while self.currentTag.name != self.ROOT_TAG_NAME:

            self.popTag()

    def reset(self):

        SGMLParser.reset(self)

        self.currentData = []

        self.currentTag = None

        self.tagStack = []

        self.pushTag(self)        

    def popTag(self):

        tag = self.tagStack.pop()

        # Tags with just one string-owning child get the child as a

        # 'string' property, so that soup.tag.string is shorthand for

        # soup.tag.contents[0]

        if len(self.currentTag.contents) == 1 and \

           isinstance(self.currentTag.contents[0], NavigableText):

            self.currentTag.string = self.currentTag.contents[0]

        #print "Pop", tag.name

        if self.tagStack:

            self.currentTag = self.tagStack[-1]

        return self.currentTag

    def pushTag(self, tag):

        #print "Push", tag.name

        if self.currentTag:

            self.currentTag.append(tag)

        self.tagStack.append(tag)

        self.currentTag = self.tagStack[-1]

    def endData(self):

        currentData = ''.join(self.currentData)

        if currentData:

            if not currentData.strip():

                if '\n' in currentData:

                    currentData = '\n'

                else:

                    currentData = ' '

            c = NavigableString

            if type(currentData) == types.UnicodeType:

                c = NavigableUnicodeString

            o = c(currentData)

            o.setup(self.currentTag, self.previous)

            if self.previous:

                self.previous.next = o

            self.previous = o

            self.currentTag.contents.append(o)

        self.currentData = []

    def _popToTag(self, name, inclusivePop=True):

        """Pops the tag stack up to and including the most recent

        instance of the given tag. If inclusivePop is false, pops the tag

        stack up to but *not* including the most recent instqance of

        the given tag."""

        if name == self.ROOT_TAG_NAME:

            return            

        #print "Pop to tag", name

        numPops = 0

        mostRecentTag = None

        for i in range(len(self.tagStack)-1, 0, -1):

            if name == self.tagStack[i].name:

                numPops = len(self.tagStack)-i

                break

        if not inclusivePop: