~launchpad-pqm/launchpad/devel

# Copyright 2009 Canonical Ltd.  This software is licensed under the

# GNU Affero General Public License version 3 (see the file LICENSE).

"""Parse Hardware Database submissions.

Base classes, intended to be used both for the commercial certification

data and for the community test submissions.

"""

__metaclass__ = type

__all__ = [

           'SubmissionParser',

           'process_pending_submissions',

          ]

import bz2

from cStringIO import StringIO

#XXX: Given the version of python we know we're running on now,

# is the try/except here necessary?

try:

    import xml.etree.cElementTree as etree

except ImportError:

    import cElementTree as etree

from datetime import datetime, timedelta

from logging import getLogger

import os

import re

import sys

import pytz

from zope.component import getUtility

from zope.interface import implements

from canonical.lazr.xml import RelaxNGValidator

from lp.services.propertycache import cachedproperty

from canonical.config import config

from canonical.librarian.interfaces import LibrarianServerError

from lp.hardwaredb.interfaces.hwdb import (

    HWBus,

    HWSubmissionProcessingStatus,

    IHWDeviceDriverLinkSet,

    IHWDeviceSet,

    IHWDriverSet,

    IHWSubmissionDeviceSet,

    IHWSubmissionSet,

    IHWVendorIDSet,

    IHWVendorNameSet,

    )

from canonical.launchpad.interfaces.launchpad import ILaunchpadCelebrities

from canonical.launchpad.interfaces.looptuner import ITunableLoop

from canonical.launchpad.utilities.looptuner import LoopTuner

from canonical.launchpad.webapp.errorlog import (

    ErrorReportingUtility,

    ScriptRequest,

    )

from lp.services.scripts.base import disable_oops_handler

_relax_ng_files = {

    '1.0': 'hardware-1_0.rng', }

_time_regex = re.compile(r"""

    ^(?P<year>\d\d\d\d)-(?P<month>\d\d)-(?P<day>\d\d)

    T(?P<hour>\d\d):(?P<minute>\d\d):(?P<second>\d\d)

    (?:\.(?P<second_fraction>\d{0,6}))?

    (?P<tz>

        (?:(?P<tz_sign>[-+])(?P<tz_hour>\d\d):(?P<tz_minute>\d\d))

        | Z)?$

    """,

    re.VERBOSE)

ROOT_UDI = '/org/freedesktop/Hal/devices/computer'

UDEV_ROOT_PATH = '/devices/LNXSYSTM:00'

# These UDIs appears in some submissions more than once.

KNOWN_DUPLICATE_UDIS = set((

    '/org/freedesktop/Hal/devices/ssb__null_',

    '/org/freedesktop/Hal/devices/uinput',

    '/org/freedesktop/Hal/devices/ignored-device',

    ))

# See include/linux/pci_ids.h in the Linux kernel sources for a complete

# list of PCI class and subclass codes.

PCI_CLASS_STORAGE = 1

PCI_SUBCLASS_STORAGE_SATA = 6

PCI_CLASS_BRIDGE = 6

PCI_SUBCLASS_BRIDGE_PCI = 4

PCI_SUBCLASS_BRIDGE_CARDBUS = 7

PCI_CLASS_SERIALBUS_CONTROLLER = 12

PCI_SUBCLASS_SERIALBUS_USB = 3

WARNING_NO_HAL_KERNEL_VERSION = 1

WARNING_NO_KERNEL_PACKAGE_DATA = 2

DB_FORMAT_FOR_VENDOR_ID = {

    'pci': '0x%04x',

    'usb_device': '0x%04x',

    'scsi': '%-8s',

    'scsi_device': '%-8s',

    }

DB_FORMAT_FOR_PRODUCT_ID = {

    'pci': '0x%04x',

    'usb_device': '0x%04x',

    'scsi': '%-16s',

    'scsi_device': '%-16s',

    }

UDEV_USB_DEVICE_PROPERTIES = set(('DEVTYPE', 'PRODUCT', 'TYPE'))

UDEV_USB_PRODUCT_RE = re.compile(

    '^[0-9a-f]{1,4}/[0-9a-f]{1,4}/[0-9a-f]{1,4}$', re.I)

UDEV_USB_TYPE_RE = re.compile('^[0-9]{1,3}/[0-9]{1,3}/[0-9]{1,3}$')

SYSFS_SCSI_DEVICE_ATTRIBUTES = set(('vendor', 'model', 'type'))

class SubmissionParser(object):

    """A Parser for the submissions to the hardware database."""

    def __init__(self, logger=None):

        if logger is None:

            logger = getLogger()

        self.logger = logger

        self.doc_parser = etree.XMLParser()

        self._logged_warnings = set()

        self.validator = {}

        directory = os.path.join(config.root, 'lib', 'canonical',

                                 'launchpad', 'scripts')

        for version, relax_ng_filename in _relax_ng_files.items():

            path = os.path.join(directory, relax_ng_filename)

            self.validator[version] = RelaxNGValidator(path)

        self._setMainSectionParsers()

        self._setHardwareSectionParsers()

        self._setSoftwareSectionParsers()

    def _logError(self, message, submission_key, create_oops=True):

        """Log `message` for an error in submission submission_key`."""

        msg = 'Parsing submission %s: %s' % (submission_key, message)

        if not create_oops:

            with disable_oops_handler(self.logger):

                self.logger.error(msg)

        else:

            self.logger.error(msg)

    def _logWarning(self, message, warning_id=None):

        """Log `message` for a warning in submission submission_key`."""

        if warning_id is None:

            issue_warning = True

        elif warning_id not in self._logged_warnings:

            issue_warning = True

            self._logged_warnings.add(warning_id)

        else:

            issue_warning = False

        if issue_warning:

            self.logger.warning(

                'Parsing submission %s: %s' % (self.submission_key, message))

    def _getValidatedEtree(self, submission, submission_key):

        """Create an etree doc from the XML string submission and validate it.

        :return: an `lxml.etree` instance representation of a valid

            submission or None for invalid submissions.

        """

        try:

            tree = etree.parse(StringIO(submission), parser=self.doc_parser)

        except SyntaxError, error_value:

            self._logError(error_value, submission_key)

            return None

        submission_doc = tree.getroot()

        if submission_doc.tag != 'system':

            self._logError("root node is not '<system>'", submission_key)

            return None

        version = submission_doc.attrib.get('version', None)

        if not version in self.validator.keys():

            self._logError(

                'invalid submission format version: %s' % repr(version),

                submission_key)

            return None

        self.submission_format_version = version

        validator = self.validator[version]

        if not validator.validate(submission):

            self._logError(

                'Relax NG validation failed.\n%s' % validator.error_log,

                submission_key,

                create_oops=False)

            return None

        return submission_doc

    def _getValueAttributeAsBoolean(self, node):

        """Return the value of the attribute "value" as a boolean."""

        value = node.attrib['value']

        # Paranoia check: The Relax NG validation already ensures that the

        # attribute value is either 'True' or 'False'.

        assert value in ('True', 'False'), (

            'Parsing submission %s: Boolean value for attribute "value" '

            'expected in tag <%s>' % (self.submission_key, node.tag))

        return value == 'True'

    def _getValueAttributeAsString(self, node):

        """Return the value of the attribute "value"."""

        # The Relax NG validation ensures that the attribute exists.

        return node.attrib['value']

    def _getValueAttributeAsDateTime(self, time_node):

        """Convert a "value" attribute into a datetime object."""

        time_text = time_node.get('value')

        # we cannot use time.strptime: this function accepts neither fractions

        # of a second nor a time zone given e.g. as '+02:30'.

        mo = _time_regex.search(time_text)

        # The Relax NG schema allows a leading minus sign and year numbers

        # with more than four digits, which are not "covered" by _time_regex.

        if mo is None:

            raise ValueError(

                'Timestamp with unreasonable value: %s' % time_text)

        time_parts = mo.groupdict()

        year = int(time_parts['year'])

        month = int(time_parts['month'])

        day = int(time_parts['day'])

        hour = int(time_parts['hour'])

        minute = int(time_parts['minute'])

        second = int(time_parts['second'])

        second_fraction = time_parts['second_fraction']

        if second_fraction is not None:

            milliseconds = second_fraction + '0' * (6 - len(second_fraction))

            milliseconds = int(milliseconds)

        else:

            milliseconds = 0

        # The Relax NG validator accepts leap seconds, but the datetime

        # constructor rejects them. The time values submitted by the HWDB

        # client are not necessarily very precise, hence we can round down

        # to 59.999999 seconds without losing any real precision.

        if second > 59:

            second = 59

            milliseconds = 999999

        timestamp = datetime(year, month, day, hour, minute, second,

                             milliseconds, tzinfo=pytz.timezone('utc'))

        tz_sign = time_parts['tz_sign']

        tz_hour = time_parts['tz_hour']

        tz_minute = time_parts['tz_minute']

        if tz_sign in ('-', '+'):

            delta = timedelta(hours=int(tz_hour), minutes=int(tz_minute))

            if tz_sign == '-':

                timestamp = timestamp + delta

            else:

                timestamp = timestamp - delta

        return timestamp

    def _getClientData(self, client_node):

        """Parse the <client> node in the <summary> section.

        :return: A dictionary with keys 'name', 'version', 'plugins'.

                 Name and version describe the client program that

                 produced the submission. Pugins is a list with one

                 entry per client plugin; each entry is dictionary with

                 the keys 'name' and 'version'.

        """

        result = {'name': client_node.get('name'),

                  'version': client_node.get('version')}

        plugins = result['plugins'] = []

        for node in client_node.getchildren():

            # Ensured by the Relax NG validation: The only allowed sub-tag

            # of <client> is <plugin>, which has the attributes 'name' and

            # 'version'.

            plugins.append({'name': node.get('name'),

                            'version': node.get('version')})

        return result

    _parse_summary_section = {

        'live_cd': _getValueAttributeAsBoolean,

        'system_id': _getValueAttributeAsString,

        'distribution': _getValueAttributeAsString,

        'distroseries': _getValueAttributeAsString,

        'architecture': _getValueAttributeAsString,

        'private': _getValueAttributeAsBoolean,

        'contactable': _getValueAttributeAsBoolean,

        'date_created': _getValueAttributeAsDateTime,

        'client': _getClientData,

        'kernel-release': _getValueAttributeAsString,

        }

    def _parseSummary(self, summary_node):

        """Parse the <summary> part of a submission.

        :return: A dictionary with the keys 'live_cd', 'system_id',

                 'distribution', 'distroseries', 'architecture',

                 'private', 'contactable', 'date_created', 'client'.

                 See the sample XML file tests/hardwaretest.xml for

                 detailed description of the values.

        """

        summary = {}

        # The Relax NG validation ensures that we have exactly those

        # sub-nodes that are listed in _parse_summary_section.

        for node in summary_node.getchildren():

            parser = self._parse_summary_section[node.tag]

            summary[node.tag] = parser(self, node)

        return summary

    def _getValueAndType(self, node):

        """Return (value, type) of a <property> or <value> node."""

        type_ = node.get('type')

        if type_ in ('dbus.Boolean', 'bool'):

            value = node.text.strip()

            # Pure paranoia: The Relax NG validation ensures that <property>

            # and <value> tags have only the allowed values.

            assert value in ('True', 'False'), (

                'Parsing submission %s: Invalid bool value for <property> or '

                    '<value>: %s' % (self.submission_key, repr(value)))

            return (value == 'True', type_)

        elif type_ in ('str', 'dbus.String', 'dbus.UTF8String'):

            return (node.text.strip(), type_)

        elif type_ in ('dbus.Byte', 'dbus.Int16', 'dbus.Int32', 'dbus.Int64',

                       'dbus.UInt16', 'dbus.UInt32', 'dbus.UInt64', 'int',

                       'long'):

            value = node.text.strip()

            return (int(value), type_)

        elif type_ in ('dbus.Double', 'float'):

            value = node.text.strip()

            return (float(value), type_)

        elif type_ in ('dbus.Array', 'list'):

            value = []

            for sub_node in node.getchildren():

                value.append(self._getValueAndType(sub_node))

            return (value, type_)

        elif type_ in ('dbus.Dictionary', 'dict'):

            value = {}

            for sub_node in node.getchildren():

                value[sub_node.get('name')] = self._getValueAndType(sub_node)

            return (value, type_)

        else:

            # This should not happen: The Relax NG validation ensures

            # that we have only those values for type_ that appear in

            # the if/elif expressions above.

            raise AssertionError(

                'Parsing submission %s: Unexpected <property> or <value> '

                    'type: %s' % (self.submission_key, type_))

    def _parseProperty(self, property_node):

        """Parse a <property> node.

        :return: (name, (value, type)) of a property.

        """

        property_name = property_node.get('name')

        return (property_node.get('name'),

                self._getValueAndType(property_node))

    def _parseProperties(self, properties_node):

        """Parse <property> sub-nodes of properties_node.

        :return: A dictionary, where each key is the name of a property;

                 the values are the tuples (value, type) of a property.

        """

        properties = {}

        for property_node in properties_node.getchildren():

            # Paranoia check: The Relax NG schema ensures that a node

            # with <property> sub-nodes has no other sub-nodes

            assert property_node.tag == 'property', (

            'Parsing submission %s: Found <%s> node, expected <property>'

                % (self.submission_key, property_node.tag))

            property_name, property_value = self._parseProperty(property_node)

            if property_name in properties.keys():

                raise ValueError(

                    '<property name="%s"> found more than once in <%s>'

                    % (property_name, properties_node.tag))

            properties[property_name] = property_value

        return properties

    def _parseDevice(self, device_node):

        """Parse a HAL <device> node.

        :return: A dictionary d with the keys 'id', 'udi', 'parent',

                 'properties'. d['id'] is an ID of the device d['udi']

                 is the HAL UDI of the device; d['properties'] is a

                 dictionary with the properties of the device (see

                 _parseProperties for details).

        """

        # The Relax NG validation ensures that the attributes "id" and

        # "udi" exist; it also ensures that "id" contains an integer.

        device_data = {'id': int(device_node.get('id')),

                       'udi': device_node.get('udi')}

        parent = device_node.get('parent', None)

        if parent is not None:

            parent = int(parent.strip())

        device_data['parent'] = parent

        device_data['properties'] = self._parseProperties(device_node)

        return device_data

    def _parseHAL(self, hal_node):

        """Parse the <hal> section of a submission.

        :return: A list, where each entry is the result of a _parseDevice

                 call.

        """

        # The Relax NG validation ensures that <hal> has the attribute

        # "version"

        hal_data = {'version': hal_node.get('version')}

        hal_data['devices'] = devices = []

        for device_node in hal_node.getchildren():

            # Pure paranoia: The Relax NG validation ensures already

            # that we have only <device> tags within <hal>

            assert device_node.tag == 'device', (

                'Parsing submission %s: Unexpected tag <%s> in <hal>'

                % (self.submission_key, device_node.tag))

            devices.append(self._parseDevice(device_node))

        return hal_data

    def _parseProcessors(self, processors_node):

        """Parse the <processors> node.

        :return: A list of dictionaries, where each dictionary d contains

                 the data of a <processor> node. The dictionary keys are

                 'id', 'name', 'properties'. d['id'] is an ID of a

                 <processor> node, d['name'] its name, and d['properties']

                 contains the properties of a processor (see

                 _parseProperties for details).

        """

        result = []

        for processor_node in processors_node.getchildren():

            # Pure paranoia: The Relax NG valiation ensures already

            # the we have only <processor> as sub-tags of <processors>.

            assert processor_node.tag == 'processor', (

                'Parsing submission %s: Unexpected tag <%s> in <processors>'

                   % (self.submission_key, processors_node.tag))

            processor = {}

            # The RelaxNG validation ensures that the attribute "id" exists

            # and that it contains an integer.

            processor['id'] = int(processor_node.get('id'))

            processor['name'] = processor_node.get('name')

            processor['properties'] = self._parseProperties(processor_node)

            result.append(processor)

        return result

    def _parseAliases(self, aliases_node):

        """Parse the <aliases> node.

        :return: A list of dictionaries, where each dictionary d has the

                 keys 'id', 'vendor', 'model'. d['id'] is the ID of a

                 HAL device; d['vendor'] is an alternative vendor name of

                 the device; d['model'] is an alternative model name.

                 See tests/hardwaretest.xml more more details.

        """

        aliases = []

        for alias_node in aliases_node.getchildren():

            # Pure paranoia: The Relax NG valiation ensures already

            # the we have only <alias> tags within <aliases>

            assert alias_node.tag == 'alias', (

                'Parsing submission %s: Unexpected tag <%s> in <aliases>'

                    % (self.submission_key, alias_node.tag))

            # The RelaxNG validation ensures that the attribute "target"

            # exists and that it contains an integer.

            alias = {'target': int(alias_node.get('target'))}

            for sub_node in alias_node.getchildren():

                # The Relax NG svalidation ensures that we have exactly

                # two subnodes: <vendor> and <model>

                alias[sub_node.tag] = sub_node.text.strip()

            aliases.append(alias)

        return aliases

    def _parseUdev(self, udev_node):

        """Parse the <udev> node.

        :return: A list of dictionaries, where each dictionary

            describes a udev device.

        The <udev> node contains the output produced by

        "udevadm info --export-db". Each entry of the dictionaries

        represents the data of the key:value pairs as they appear

        in this data. The value of d['S'] is a list of strings,

        the value s['E'] is a dictionary containing the key=value

        pairs of the "E:" lines.

        """

        # We get the plain text as produced by "udevadm info --export-db"

        # This data looks like:

        #

        # P: /devices/LNXSYSTM:00

        # E: UDEV_LOG=3

        # E: DEVPATH=/devices/LNXSYSTM:00

        # E: MODALIAS=acpi:LNXSYSTM:

        #

        # P: /devices/LNXSYSTM:00/ACPI_CPU:00

        # E: UDEV_LOG=3

        # E: DEVPATH=/devices/LNXSYSTM:00/ACPI_CPU:00

        # E: DRIVER=processor

        # E: MODALIAS=acpi:ACPI_CPU:

        #

        # Data for different devices is separated by empty lines.

        # Each line for a device consists of key:value pairs.

        # The following keys are defined:

        #

        # A: udev_device_get_num_fake_partitions()

        # E: udev_device_get_properties_list_entry()

        # L: the device link priority (udev_device_get_devlink_priority())

        # N: the device node file name (udev_device_get_devnode())

        # P: the device path (udev_device_get_devpath())

        # R: udev_device_get_ignore_remove()

        # S: udev_get_dev_path()

        # W: udev_device_get_watch_handle()

        #

        # The key P is always present; the keys A, L, N, R, W appear at

        # most once per device; the keys E and S may appear more than

        # once.

        # The values of the E records have the format "key=value"

        #

        # See also the libudev reference manual:

        # http://www.kernel.org/pub/linux/utils/kernel/hotplug/libudev/

        # and the udev file udevadm-info.c, function print_record()

        udev_data = udev_node.text.split('\n')

        devices = []

        device = None

        line_number = 0

        device_id = 0

        for line_number, line in enumerate(udev_data):

            if len(line) == 0:

                device = None

                continue

            record = line.split(':', 1)

            if len(record) != 2:

                self._logError(

                    'Line %i in <udev>: No valid key:value data: %r'

                    % (line_number, line),

                    self.submission_key)

                return None

            key, value = record

            if device is None:

                device_id += 1

                device = {

                    'E': {},

                    'S': [],

                    'id': device_id,

                    }

                devices.append(device)

            # Some attribute lines have a space character after the

            # ':', others don't have it (see udevadm-info.c).

            value = value.lstrip()

            if key == 'E':

                property_data = value.split('=', 1)

                if len(property_data) != 2:

                    self._logError(

                        'Line %i in <udev>: Property without valid key=value '

                        'data: %r' % (line_number, line),

                        self.submission_key)

                    return None

                property_key, property_value = property_data

                device['E'][property_key] = property_value

            elif key == 'S':

                device['S'].append(value)

            else:

                if key in device:

                    self._logWarning(

                        'Line %i in <udev>: Duplicate attribute key: %r'

                        % (line_number, line),

                        self.submission_key)

                device[key] = value

        return devices

    def _parseDmi(self, dmi_node):

        """Parse the <dmi> node.

        :return: A dictionary containing the key:value pairs of the DMI data.

        """

        dmi_data = {}

        dmi_text = dmi_node.text.strip().split('\n')

        for line_number, line in enumerate(dmi_text):

            record = line.split(':', 1)

            if len(record) != 2:

                self._logError(

                    'Line %i in <dmi>: No valid key:value data: %r'

                    % (line_number, line),

                    self.submission_key)

                return None

            dmi_data[record[0]] = record[1]

        return dmi_data

    def _parseSysfsAttributes(self, sysfs_node):

        """Parse the <sysfs-attributes> node.

        :return: A dictionary {path: attrs, ...} where path is the

            path is the path of a sysfs directory, and where attrs

            is a dictionary containing attribute names and values.

        A sample of the input data:

        P: /devices/LNXSYSTM:00/LNXPWRBN:00/input/input0

        A: modalias=input:b0019v0000p0001e0000-e0,1,k74,ramlsfw

        A: uniq=

        A: phys=LNXPWRBN/button/input0

        A: name=Power Button

        P: /devices/LNXSYSTM:00/device:00/PNP0A08:00/device:03/input/input8

        A: modalias=input:b0019v0000p0006e0000-e0,1,kE0,E1,E3,F0,F1

        A: uniq=

        A: phys=/video/input0

        A: name=Video Bus

        Data for different devices is separated by empty lines. The data

        for each device starts with a line 'P: /devices/LNXSYSTM...',

        specifying the sysfs path of a device, followed by zero or more

        lines of the form 'A: key=value'

        """

        sysfs_lines = sysfs_node.text.split('\n')

        sysfs_data = {}

        attributes = None

        for line_number, line in enumerate(sysfs_lines):

            if len(line) == 0:

                attributes = None

                continue

            record = line.split(': ', 1)

            if len(record) != 2:

                self._logError(

                    'Line %i in <sysfs-attributes>: No valid key:value data: '

                    '%r' % (line_number, line),

                    self.submission_key)

                return None

            key, value = record

            if key == 'P':

                if attributes is not None:

                    self._logError(

                        "Line %i in <sysfs-attributes>: duplicate 'P' line "

                        "found: %r" % (line_number, line),

                        self.submission_key)

                    return None

                attributes = {}

                sysfs_data[value] = attributes

            elif key == 'A':

                if attributes is None:

                    self._logError(

                        "Line %i in <sysfs-attributes>: Block for a device "

                        "does not start with 'P:': %r" % (line_number, line),

                        self.submission_key)

                    return None

                attribute_data = value.split('=', 1)

                if len(attribute_data) != 2:

                    self._logError(

                        'Line %i in <sysfs-attributes>: Attribute line does '

                        'not contain key=value data: %r'

                        % (line_number, line),

                        self.submission_key)

                    return None

                attributes[attribute_data[0]] = attribute_data[1]

            else:

                self._logError(

                    'Line %i in <sysfs-attributes>: Unexpected key: %r'

                    % (line_number, line),

                    self.submission_key)

                return None

        return sysfs_data

    def _setHardwareSectionParsers(self):

        self._parse_hardware_section = {

            'hal': self._parseHAL,

            'processors': self._parseProcessors,

            'aliases': self._parseAliases,

            'udev': self._parseUdev,

            'dmi': self._parseDmi,

            'sysfs-attributes': self._parseSysfsAttributes,

            }

    def _parseHardware(self, hardware_node):

        """Parse the <hardware> part of a submission.

        :return: A dictionary with the keys 'hal', 'processors', 'aliases',

                 where the values are the parsing results of _parseHAL,

                 _parseProcessors, _parseAliases.

        """

        hardware_data = {}

        for node in hardware_node.getchildren():

            parser = self._parse_hardware_section[node.tag]

            result = parser(node)

            if result is None:

                return None

            hardware_data[node.tag] = result

        return hardware_data

    def _parseLSBRelease(self, lsb_node):

        """Parse the <lsb_release> part of a submission.

        :return: A dictionary with the content of the <properta> nodes

                 within the <lsb> node. See tests/hardwaretest.xml for

                 details.

        """

        return self._parseProperties(lsb_node)

    def _parsePackages(self, packages_node):

        """Parse the <packages> part of a submission.

        :return: A dictionary with one entry per <package> sub-node.

                 The key is the package name, the value a dictionary

                 containing the content of the <property> nodes within

                 <package>. See tests/hardwaretest.xml for more details.

        """

        packages = {}

        for package_node in packages_node.getchildren():

            # Pure paranoia: The Relax NG validation ensures already

            # that we have only <package> tags within <packages>.

            assert package_node.tag == 'package', (

                'Parsing submission %s: Unexpected tag <%s> in <packages>'

                % (self.submission_key, package_node.tag))

            package_name = package_node.get('name')

            if package_name in packages.keys():

                raise ValueError(

                    '<package name="%s"> appears more than once in <packages>'

                    % package_name)

            # The RelaxNG validation ensures that the attribute "id" exists

            # and that it contains an integer.

            package_data = {'id': int(package_node.get('id'))}

            package_data['properties'] = self._parseProperties(package_node)

            packages[package_name] = package_data

        return packages

    def _parseXOrg(self, xorg_node):

        """Parse the <xorg> part of a submission.

        :return: A dictionary with the keys 'version' and 'drivers'.

                 d['version'] is the xorg version; d['drivers'] is

                 a dictionary with one entry for each <driver> sub-node,

                 where the key is the driver name, the value is a dictionary

                 containing the attributes of the <driver> node. See

                 tests/hardwaretest.xml for more details.

        """

        xorg_data = {'version': xorg_node.get('version')}

        xorg_data['drivers'] = xorg_drivers = {}

        for driver_node in xorg_node.getchildren():

            # Pure paranoia: The Relax NG validation ensures already