lxml.html

Package lxml :: Package html |
[hide private]
[frames] | no frames]

# Source Code for Package lxml.html

       1  """The ``lxml.html`` tool set for HTML handling.
       2  """
       3
       4  import threading
       5  import re
       6  try:
       7      from urlparse import urljoin
       8  except ImportError:
       9      # Python 3
      10      from urllib.parse import urljoin
      11  import copy
      12  from lxml import etree
      13  from lxml.html import defs
      14  from lxml import cssselect
      15  from lxml.html._setmixin import SetMixin
      16  try:
      17      from UserDict import DictMixin
      18  except ImportError:
      19      # DictMixin was introduced in Python 2.4
      20      from lxml.html._dictmixin import DictMixin
      21  try:
      22      set
      23  except NameError:
      24      # Python 2.3
      25      from sets import Set as set
      26  try:
      27      bytes = __builtins__["bytes"]
      28  except (KeyError, NameError):
      29      # Python < 2.6
      30      bytes = str
      31  try:
      32      unicode = __builtins__["unicode"]
      33  except (KeyError, NameError):
      34      # Python 3
      35      unicode = str
      36  try:
      37      basestring = __builtins__["basestring"]
      38  except (KeyError, NameError):
      39      # Python 3
      40      basestring = (str, bytes)
      41
      42 -def __fix_docstring(s):
      43      if not s:
      44          return s
      45      import sys
      46      if sys.version_info[0] >= 3:
      47          sub = re.compile(r"^(\s*)u'", re.M).sub
      48      else:
      49          sub = re.compile(r"^(\s*)b'", re.M).sub
      50      return sub(r"\1'", s)
      51
      52  __all__ = [
      53      'document_fromstring', 'fragment_fromstring', 'fragments_fromstring', 'fromstring',
      54      'tostring', 'Element', 'defs', 'open_in_browser', 'submit_form',
      55      'find_rel_links', 'find_class', 'make_links_absolute',
      56      'resolve_base_href', 'iterlinks', 'rewrite_links', 'open_in_browser', 'parse']
      57
      58  XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
      59
      60  _rel_links_xpath = etree.XPath("descendant-or-self::a[@rel]|descendant-or-self::x:a[@rel]",
      61                                 namespaces={'x':XHTML_NAMESPACE})
      62  _options_xpath = etree.XPath("descendant-or-self::option|descendant-or-self::x:option",
      63                               namespaces={'x':XHTML_NAMESPACE})
      64  _forms_xpath = etree.XPath("descendant-or-self::form|descendant-or-self::x:form",
      65                             namespaces={'x':XHTML_NAMESPACE})
      66  #_class_xpath = etree.XPath(r"descendant-or-self::*[regexp:match(@class, concat('\b', $class_name, '\b'))]", {'regexp': 'http://exslt.org/regular-expressions'})
      67  _class_xpath = etree.XPath("descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), concat(' ', $class_name, ' '))]")
      68  _id_xpath = etree.XPath("descendant-or-self::*[@id=$id]")
      69  _collect_string_content = etree.XPath("string()")
      70  _css_url_re = re.compile(r'url\(('+'["][^"]*["]|'+"['][^']*[']|"+r'[^)]*)\)', re.I)
      71  _css_import_re = re.compile(r'@import "(.*?)"')
      72  _label_xpath = etree.XPath("//label[@for=$id]|//x:label[@for=$id]",
      73                             namespaces={'x':XHTML_NAMESPACE})
      74  _archive_re = re.compile(r'[^ ]+')
      75
      76 -def _unquote_match(s, pos):
      77      if s[:1] == '"' and s[-1:] == '"' or s[:1] == "'" and s[-1:] == "'":
      78          return s[1:-1], pos+1
      79      else:
      80          return s,pos
      81
      82 -def _transform_result(typ, result):
      83      """Convert the result back into the input type.
      84      """
      85      if issubclass(typ, bytes):
      86          return tostring(result, encoding='utf-8')
      87      elif issubclass(typ, unicode):
      88          return tostring(result, encoding=unicode)
      89      else:
      90          return result
      91
      92 -def _nons(tag):
      93      if isinstance(tag, basestring):
      94          if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
      95              return tag.split('}')[-1]
      96      return tag
      97
      98 -class HtmlMixin(object):
      99
     100 -    def base_url(self):
     101          """
     102          Returns the base URL, given when the page was parsed.
     103
     104          Use with ``urlparse.urljoin(el.base_url, href)`` to get
     105          absolute URLs.
     106          """
     107          return self.getroottree().docinfo.URL
     108      base_url = property(base_url, doc=base_url.__doc__)
     109
     110 -    def forms(self):
     111          """
     112          Return a list of all the forms
     113          """
     114          return _forms_xpath(self)
     115      forms = property(forms, doc=forms.__doc__)
     116
     117 -    def body(self):
     118          """
     119          Return the <body> element.  Can be called from a child element
     120          to get the document's head.
     121          """
     122          return self.xpath('//body|//x:body', namespaces={'x':XHTML_NAMESPACE})[0]
     123      body = property(body, doc=body.__doc__)
     124
     125 -    def head(self):
     126          """
     127          Returns the <head> element.  Can be called from a child
     128          element to get the document's head.
     129          """
     130          return self.xpath('//head|//x:head', namespaces={'x':XHTML_NAMESPACE})[0]
     131      head = property(head, doc=head.__doc__)
     132
     133 -    def _label__get(self):
     134          """
     135          Get or set any <label> element associated with this element.
     136          """
     137          id = self.get('id')
     138          if not id:
     139              return None
     140          result = _label_xpath(self, id=id)
     141          if not result:
     142              return None
     143          else:
     144              return result[0]
     145 -    def _label__set(self, label):
     146          id = self.get('id')
     147          if not id:
     148              raise TypeError(
     149                  "You cannot set a label for an element (%r) that has no id"
     150                  % self)
     151          if _nons(label.tag) != 'label':
     152              raise TypeError(
     153                  "You can only assign label to a label element (not %r)"
     154                  % label)
     155          label.set('for', id)
     156 -    def _label__del(self):
     157          label = self.label
     158          if label is not None:
     159              del label.attrib['for']
     160      label = property(_label__get, _label__set, _label__del, doc=_label__get.__doc__)
     161
     162 -    def drop_tree(self):
     163          """
     164          Removes this element from the tree, including its children and
     165          text.  The tail text is joined to the previous element or
     166          parent.
     167          """
     168          parent = self.getparent()
     169          assert parent is not None
     170          if self.tail:
     171              previous = self.getprevious()
     172              if previous is None:
     173                  parent.text = (parent.text or '') + self.tail
     174              else:
     175                  previous.tail = (previous.tail or '') + self.tail
     176          parent.remove(self)
     177
     178 -    def drop_tag(self):
     179          """
     180          Remove the tag, but not its children or text.  The children and text
     181          are merged into the parent.
     182
     183          Example::
     184
     185              >>> h = fragment_fromstring('<div>Hello <b>World!</b></div>')
     186              >>> h.find('.//b').drop_tag()
     187              >>> print(tostring(h, encoding=unicode))
     188              <div>Hello World!</div>
     189          """
     190          parent = self.getparent()
     191          assert parent is not None
     192          previous = self.getprevious()
     193          if self.text and isinstance(self.tag, basestring):
     194              # not a Comment, etc.
     195              if previous is None:
     196                  parent.text = (parent.text or '') + self.text
     197              else:
     198                  previous.tail = (previous.tail or '') + self.text
     199          if self.tail:
     200              if len(self):
     201                  last = self[-1]
     202                  last.tail = (last.tail or '') + self.tail
     203              elif previous is None:
     204                  parent.text = (parent.text or '') + self.tail
     205              else:
     206                  previous.tail = (previous.tail or '') + self.tail
     207          index = parent.index(self)
     208          parent[index:index+1] = self[:]
     209
     210 -    def find_rel_links(self, rel):
     211          """
     212          Find any links like ``<a rel="{rel}">...</a>``; returns a list of elements.
     213          """
     214          rel = rel.lower()
     215          return [el for el in _rel_links_xpath(self)
     216                  if el.get('rel').lower() == rel]
     217
     218 -    def find_class(self, class_name):
     219          """
     220          Find any elements with the given class name.
     221          """
     222          return _class_xpath(self, class_name=class_name)
     223
     224 -    def get_element_by_id(self, id, *default):
     225          """
     226          Get the first element in a document with the given id.  If none is
     227          found, return the default argument if provided or raise KeyError
     228          otherwise.
     229
     230          Note that there can be more than one element with the same id,
     231          and this isn't uncommon in HTML documents found in the wild.
     232          Browsers return only the first match, and this function does
     233          the same.
     234          """
     235          try:
     236              # FIXME: should this check for multiple matches?
     237              # browsers just return the first one
     238              return _id_xpath(self, id=id)[0]
     239          except IndexError:
     240              if default:
     241                  return default[0]
     242              else:
     243                  raise KeyError(id)
     244
     245 -    def text_content(self):
     246          """
     247          Return the text content of the tag (and the text in any children).
     248          """
     249          return _collect_string_content(self)
     250
     251 -    def cssselect(self, expr):
     252          """
     253          Run the CSS expression on this element and its children,
     254          returning a list of the results.
     255
     256          Equivalent to lxml.cssselect.CSSSelect(expr)(self) -- note
     257          that pre-compiling the expression can provide a substantial
     258          speedup.
     259          """
     260          return cssselect.CSSSelector(expr)(self)
     261
     262      ########################################
     263      ## Link functions
     264      ########################################
     265
     266 -    def make_links_absolute(self, base_url=None, resolve_base_href=True):
     267          """
     268          Make all links in the document absolute, given the
     269          ``base_url`` for the document (the full URL where the document
     270          came from), or if no ``base_url`` is given, then the ``.base_url`` of the document.
     271
     272          If ``resolve_base_href`` is true, then any ``<base href>``
     273          tags in the document are used *and* removed from the document.
     274          If it is false then any such tag is ignored.
     275          """
     276          if base_url is None:
     277              base_url = self.base_url
     278              if base_url is None:
     279                  raise TypeError(
     280                      "No base_url given, and the document has no base_url")
     281          if resolve_base_href:
     282              self.resolve_base_href()
     283          def link_repl(href):
     284              return urljoin(base_url, href)
     285          self.rewrite_links(link_repl)
     286
     287 -    def resolve_base_href(self):
     288          """
     289          Find any ``<base href>`` tag in the document, and apply its
     290          values to all links found in the document.  Also remove the
     291          tag once it has been applied.
     292          """
     293          base_href = None
     294          basetags = self.xpath('//base[@href]|//x:base[@href]', namespaces={'x':XHTML_NAMESPACE})
     295          for b in basetags:
     296              base_href = b.get('href')
     297              b.drop_tree()
     298          if not base_href:
     299              return
     300          self.make_links_absolute(base_href, resolve_base_href=False)
     301
     302 -    def iterlinks(self):
     303          """
     304          Yield (element, attribute, link, pos), where attribute may be None
     305          (indicating the link is in the text).  ``pos`` is the position
     306          where the link occurs; often 0, but sometimes something else in
     307          the case of links in stylesheets or style tags.
     308
     309          Note: <base href> is *not* taken into account in any way.  The
     310          link you get is exactly the link in the document.
     311          """
     312          link_attrs = defs.link_attrs
     313          for el in self.iter():
     314              attribs = el.attrib
     315              tag = _nons(el.tag)
     316              if tag != 'object':
     317                  for attrib in link_attrs:
     318                      if attrib in attribs:
     319                          yield (el, attrib, attribs[attrib], 0)
     320              elif tag == 'object':
     321                  codebase = None
     322                  ## <object> tags have attributes that are relative to
     323                  ## codebase
     324                  if 'codebase' in attribs:
     325                      codebase = el.get('codebase')
     326                      yield (el, 'codebase', codebase, 0)
     327                  for attrib in 'classid', 'data':
     328                      if attrib in attribs:
     329                          value = el.get(attrib)
     330                          if codebase is not None:
     331                              value = urljoin(codebase, value)
     332                          yield (el, attrib, value, 0)
     333                  if 'archive' in attribs:
     334                      for match in _archive_re.finditer(el.get('archive')):
     335                          value = match.group(0)
     336                          if codebase is not None:
     337                              value = urljoin(codebase, value)
     338                          yield (el, 'archive', value, match.start())
     339              if tag == 'param':
     340                  valuetype = el.get('valuetype') or ''
     341                  if valuetype.lower() == 'ref':
     342                      ## FIXME: while it's fine we *find* this link,
     343                      ## according to the spec we aren't supposed to
     344                      ## actually change the value, including resolving
     345                      ## it.  It can also still be a link, even if it
     346                      ## doesn't have a valuetype="ref" (which seems to be the norm)
     347                      ## http://www.w3.org/TR/html401/struct/objects.html#adef-valuetype
     348                      yield (el, 'value', el.get('value'), 0)
     349              if tag == 'style' and el.text:
     350                  for match in _css_url_re.finditer(el.text):
     351                      url, start = _unquote_match(match.group(1), match.start(1))
     352                      yield (el, None, url, start)
     353                  for match in _css_import_re.finditer(el.text):
     354                      yield (el, None, match.group(1), match.start(1))
     355              if 'style' in attribs:
     356                  for match in _css_url_re.finditer(attribs['style']):
     357                      url, start = _unquote_match(match.group(1), match.start(1))
     358                      yield (el, 'style', url, start)
     359
     360 -    def rewrite_links(self, link_repl_func, resolve_base_href=True,
     361                        base_href=None):
     362          """
     363          Rewrite all the links in the document.  For each link
     364          ``link_repl_func(link)`` will be called, and the return value
     365          will replace the old link.
     366
     367          Note that links may not be absolute (unless you first called
     368          ``make_links_absolute()``), and may be internal (e.g.,
     369          ``'#anchor'``).  They can also be values like
     370          ``'mailto:email'`` or ``'javascript:expr'``.
     371
     372          If you give ``base_href`` then all links passed to
     373          ``link_repl_func()`` will take that into account.
     374
     375          If the ``link_repl_func`` returns None, the attribute or
     376          tag text will be removed completely.
     377          """
     378          if base_href is not None:
     379              # FIXME: this can be done in one pass with a wrapper
     380              # around link_repl_func
     381              self.make_links_absolute(base_href, resolve_base_href=resolve_base_href)
     382          elif resolve_base_href:
     383              self.resolve_base_href()
     384          for el, attrib, link, pos in self.iterlinks():
     385              new_link = link_repl_func(link.strip())
     386              if new_link == link:
     387                  continue
     388              if new_link is None:
     389                  # Remove the attribute or element content
     390                  if attrib is None:
     391                      el.text = ''
     392                  else:
     393                      del el.attrib[attrib]
     394                  continue
     395              if attrib is None:
     396                  new = el.text[:pos] + new_link + el.text[pos+len(link):]
     397                  el.text = new
     398              else:
     399                  cur = el.attrib[attrib]
     400                  if not pos and len(cur) == len(link):
     401                      # Most common case
     402                      el.attrib[attrib] = new_link
     403                  else:
     404                      new = cur[:pos] + new_link + cur[pos+len(link):]
     405                      el.attrib[attrib] = new
     406
     407
     408 -class _MethodFunc(object):
     409      """
     410      An object that represents a method on an element as a function;
     411      the function takes either an element or an HTML string.  It
     412      returns whatever the function normally returns, or if the function
     413      works in-place (and so returns None) it returns a serialized form
     414      of the resulting document.
     415      """
     416 -    def __init__(self, name, copy=False, source_class=HtmlMixin):
     417          self.name = name
     418          self.copy = copy
     419          self.__doc__ = getattr(source_class, self.name).__doc__
     420 -    def __call__(self, doc, *args, **kw):
     421          result_type = type(doc)
     422          if isinstance(doc, basestring):
     423              if 'copy' in kw:
     424                  raise TypeError(
     425                      "The keyword 'copy' can only be used with element inputs to %s, not a string input" % self.name)
     426              doc = fromstring(doc, **kw)
     427          else:
     428              if 'copy' in kw:
     429                  copy = kw.pop('copy')
     430              else:
     431                  copy = self.copy
     432              if copy:
     433                  doc = copy.deepcopy(doc)
     434          meth = getattr(doc, self.name)
     435          result = meth(*args, **kw)
     436          # FIXME: this None test is a bit sloppy
     437          if result is None:
     438              # Then return what we got in
     439              return _transform_result(result_type, doc)
     440          else:
     441              return result
     442
     443  find_rel_links = _MethodFunc('find_rel_links', copy=False)
     444  find_class = _MethodFunc('find_class', copy=False)
     445  make_links_absolute = _MethodFunc('make_links_absolute', copy=True)
     446  resolve_base_href = _MethodFunc('resolve_base_href', copy=True)
     447  iterlinks = _MethodFunc('iterlinks', copy=False)
     448  rewrite_links = _MethodFunc('rewrite_links', copy=True)
     449
     450 -class HtmlComment(etree.CommentBase, HtmlMixin):
     451      pass
     452
     453 -class HtmlElement(etree.ElementBase, HtmlMixin):
     454      pass
     455
     456 -class HtmlProcessingInstruction(etree.PIBase, HtmlMixin):
     457      pass
     458
     459 -class HtmlEntity(etree.EntityBase, HtmlMixin):
     460      pass
     461
     462
     463 -class HtmlElementClassLookup(etree.CustomElementClassLookup):
     464      """A lookup scheme for HTML Element classes.
     465
     466      To create a lookup instance with different Element classes, pass a tag
     467      name mapping of Element classes in the ``classes`` keyword argument and/or
     468      a tag name mapping of Mixin classes in the ``mixins`` keyword argument.
     469      The special key '*' denotes a Mixin class that should be mixed into all
     470      Element classes.
     471      """
     472      _default_element_classes = {}
     473
     474 -    def __init__(self, classes=None, mixins=None):
     475          etree.CustomElementClassLookup.__init__(self)
     476          if classes is None:
     477              classes = self._default_element_classes.copy()
     478          if mixins:
     479              mixers = {}
     480              for name, value in mixins:
     481                  if name == '*':
     482                      for n in classes.keys():
     483                          mixers.setdefault(n, []).append(value)
     484                  else:
     485                      mixers.setdefault(name, []).append(value)
     486              for name, mix_bases in mixers.items():
     487                  cur = classes.get(name, HtmlElement)
     488                  bases = tuple(mix_bases + [cur])
     489                  classes[name] = type(cur.__name__, bases, {})
     490          self._element_classes = classes
     491
     492 -    def lookup(self, node_type, document, namespace, name):
     493          if node_type == 'element':
     494              return self._element_classes.get(name.lower(), HtmlElement)
     495          elif node_type == 'comment':
     496              return HtmlComment
     497          elif node_type == 'PI':
     498              return HtmlProcessingInstruction
     499          elif node_type == 'entity':
     500              return HtmlEntity
     501          # Otherwise normal lookup
     502          return None
     503
     504  ################################################################################
     505  # parsing
     506  ################################################################################
     507
     508 -def document_fromstring(html, parser=None, **kw):
     509      if parser is None:
     510          parser = html_parser
     511      value = etree.fromstring(html, parser, **kw)
     512      if value is None:
     513          raise etree.ParserError(
     514              "Document is empty")
     515      return value
     516
     517 -def fragments_fromstring(html, no_leading_text=False, base_url=None,
     518                           parser=None, **kw):
     519      """
     520      Parses several HTML elements, returning a list of elements.
     521
     522      The first item in the list may be a string (though leading
     523      whitespace is removed).  If no_leading_text is true, then it will
     524      be an error if there is leading text, and it will always be a list
     525      of only elements.
     526
     527      base_url will set the document's base_url attribute (and the tree's docinfo.URL)
     528      """
     529      if parser is None:
     530          parser = html_parser
     531      # FIXME: check what happens when you give html with a body, head, etc.
     532      start = html[:20].lstrip().lower()
     533      if not start.startswith('<html') and not start.startswith('<!doctype'):
     534          html = '<html><body>%s</body></html>' % html
     535      doc = document_fromstring(html, parser=parser, base_url=base_url, **kw)
     536      assert _nons(doc.tag) == 'html'
     537      bodies = [e for e in doc if _nons(e.tag) == 'body']
     538      assert len(bodies) == 1, ("too many bodies: %r in %r" % (bodies, html))
     539      body = bodies[0]
     540      elements = []
     541      if no_leading_text and body.text and body.text.strip():
     542          raise etree.ParserError(
     543              "There is leading text: %r" % body.text)
     544      if body.text and body.text.strip():
     545          elements.append(body.text)
     546      elements.extend(body)
     547      # FIXME: removing the reference to the parent artificial document
     548      # would be nice
     549      return elements
     550
     551 -def fragment_fromstring(html, create_parent=False, base_url=None,
     552                          parser=None, **kw):
     553      """
     554      Parses a single HTML element; it is an error if there is more than
     555      one element, or if anything but whitespace precedes or follows the
     556      element.
     557
     558      If create_parent is true (or is a tag name) then a parent node
     559      will be created to encapsulate the HTML in a single element.
     560
     561      base_url will set the document's base_url attribute (and the tree's docinfo.URL)
     562      """
     563      if parser is None:
     564          parser = html_parser
     565      if create_parent:
     566          if not isinstance(create_parent, basestring):
     567              create_parent = 'div'
     568          return fragment_fromstring('<%s>%s</%s>' % (
     569              create_parent, html, create_parent),
     570                                     parser=parser, base_url=base_url, **kw)
     571      elements = fragments_fromstring(html, parser=parser, no_leading_text=True,
     572                                      base_url=base_url, **kw)
     573      if not elements:
     574          raise etree.ParserError(
     575              "No elements found")
     576      if len(elements) > 1:
     577          raise etree.ParserError(
     578              "Multiple elements found (%s)"
     579              % ', '.join([_element_name(e) for e in elements]))
     580      el = elements[0]
     581      if el.tail and el.tail.strip():
     582          raise etree.ParserError(
     583              "Element followed by text: %r" % el.tail)
     584      el.tail = None
     585      return el
     586
     587 -def fromstring(html, base_url=None, parser=None, **kw):
     588      """
     589      Parse the html, returning a single element/document.
     590
     591      This tries to minimally parse the chunk of text, without knowing if it
     592      is a fragment or a document.
     593
     594      base_url will set the document's base_url attribute (and the tree's docinfo.URL)
     595      """
     596      if parser is None:
     597          parser = html_parser
     598      start = html[:10].lstrip().lower()
     599      if start.startswith('<html') or start.startswith('<!doctype'):
     600          # Looks like a full HTML document
     601          return document_fromstring(html, parser=parser, base_url=base_url, **kw)
     602      # otherwise, lets parse it out...
     603      doc = document_fromstring(html, parser=parser, base_url=base_url, **kw)
     604      bodies = doc.findall('body')
     605      if not bodies:
     606          bodies = doc.findall('{%s}body' % XHTML_NAMESPACE)
     607      if bodies:
     608          body = bodies[0]
     609          if len(bodies) > 1:
     610              # Somehow there are multiple bodies, which is bad, but just
     611              # smash them into one body
     612              for other_body in bodies[1:]:
     613                  if other_body.text:
     614                      if len(body):
     615                          body[-1].tail = (body[-1].tail or '') + other_body.text
     616                      else:
     617                          body.text = (body.text or '') + other_body.text
     618                  body.extend(other_body)
     619                  # We'll ignore tail
     620                  # I guess we are ignoring attributes too
     621                  other_body.drop_tree()
     622      else:
     623          body = None
     624      heads = doc.findall('head')
     625      if not heads:
     626          heads = doc.findall('{%s}head' % XHTML_NAMESPACE)
     627      if heads:
     628          # Well, we have some sort of structure, so lets keep it all
     629          head = heads[0]
     630          if len(heads) > 1:
     631              for other_head in heads[1:]:
     632                  head.extend(other_head)
     633                  # We don't care about text or tail in a head
     634                  other_head.drop_tree()
     635          return doc
     636      if (len(body) == 1 and (not body.text or not body.text.strip())
     637          and (not body[-1].tail or not body[-1].tail.strip())):
     638          # The body has just one element, so it was probably a single
     639          # element passed in
     640          return body[0]
     641      # Now we have a body which represents a bunch of tags which have the
     642      # content that was passed in.  We will create a fake container, which
     643      # is the body tag, except <body> implies too much structure.
     644      if _contains_block_level_tag(body):
     645          body.tag = 'div'
     646      else:
     647          body.tag = 'span'
     648      return body
     649
     650 -def parse(filename_or_url, parser=None, base_url=None, **kw):
     651      """
     652      Parse a filename, URL, or file-like object into an HTML document
     653      tree.  Note: this returns a tree, not an element.  Use
     654      ``parse(...).getroot()`` to get the document root.
     655
     656      You can override the base URL with the ``base_url`` keyword.  This
     657      is most useful when parsing from a file-like object.
     658      """
     659      if parser is None:
     660          parser = html_parser
     661      return etree.parse(filename_or_url, parser, base_url=base_url, **kw)
     662
     663 -def _contains_block_level_tag(el):
     664      # FIXME: I could do this with XPath, but would that just be
     665      # unnecessarily slow?
     666      for el in el.iter():
     667          if _nons(el.tag) in defs.block_tags:
     668              return True
     669      return False
     670
     671 -def _element_name(el):
     672      if isinstance(el, etree.CommentBase):
     673          return 'comment'
     674      elif isinstance(el, basestring):
     675          return 'string'
     676      else:
     677          return _nons(el.tag)
     678
     679  ################################################################################
     680  # form handling
     681  ################################################################################
     682
     683 -class FormElement(HtmlElement):
     684      """
     685      Represents a <form> element.
     686      """
     687
     688 -    def inputs(self):
     689          """
     690          Returns an accessor for all the input elements in the form.
     691
     692          See `InputGetter` for more information about the object.
     693          """
     694          return InputGetter(self)
     695      inputs = property(inputs, doc=inputs.__doc__)
     696
     697 -    def _fields__get(self):
     698          """
     699          Dictionary-like object that represents all the fields in this
     700          form.  You can set values in this dictionary to effect the
     701          form.
     702          """
     703          return FieldsDict(self.inputs)
     704 -    def _fields__set(self, value):
     705          prev_keys = self.fields.keys()
     706          for key, value in value.iteritems():
     707              if key in prev_keys:
     708                  prev_keys.remove(key)
     709              self.fields[key] = value
     710          for key in prev_keys:
     711              if key is None:
     712                  # Case of an unnamed input; these aren't really
     713                  # expressed in form_values() anyway.
     714                  continue
     715              self.fields[key] = None
     716
     717      fields = property(_fields__get, _fields__set, doc=_fields__get.__doc__)
     718
     719 -    def _name(self):
     720          if self.get('name'):
     721              return self.get('name')
     722          elif self.get('id'):
     723              return '#' + self.get('id')
     724          forms = list(self.body.iter('form'))
     725          if not forms:
     726              forms = list(self.body.iter('{%s}form' % XHTML_NAMESPACE))
     727          return str(forms.index(self))
     728
     729 -    def form_values(self):
     730          """
     731          Return a list of tuples of the field values for the form.
     732          This is suitable to be passed to ``urllib.urlencode()``.
     733          """
     734          results = []
     735          for el in self.inputs:
     736              name = el.name
     737              if not name:
     738                  continue
     739              tag = _nons(el.tag)
     740              if tag == 'textarea':
     741                  results.append((name, el.value))
     742              elif tag == 'select':
     743                  value = el.value
     744                  if el.multiple:
     745                      for v in value:
     746                          results.append((name, v))
     747                  elif value is not None:
     748                      results.append((name, el.value))
     749              else:
     750                  assert tag == 'input', (
     751                      "Unexpected tag: %r" % el)
     752                  if el.checkable and not el.checked:
     753                      continue
     754                  if el.type in ('submit', 'image', 'reset'):
     755                      continue
     756                  value = el.value
     757                  if value is not None:
     758                      results.append((name, el.value))
     759          return results
     760
     761 -    def _action__get(self):
     762          """
     763          Get/set the form's ``action`` attribute.
     764          """
     765          base_url = self.base_url
     766          action = self.get('action')
     767          if base_url and action is not None:
     768              return urljoin(base_url, action)
     769          else:
     770              return action
     771 -    def _action__set(self, value):
     772          self.set('action', value)
     773 -    def _action__del(self):
     774          if 'action' in self.attrib:
     775              del self.attrib['action']
     776      action = property(_action__get, _action__set, _action__del, doc=_action__get.__doc__)
     777
     778 -    def _method__get(self):
     779          """
     780          Get/set the form's method.  Always returns a capitalized
     781          string, and defaults to ``'GET'``
     782          """
     783          return self.get('method', 'GET').upper()
     784 -    def _method__set(self, value):
     785          self.set('method', value.upper())
     786      method = property(_method__get, _method__set, doc=_method__get.__doc__)
     787
     788  HtmlElementClassLookup._default_element_classes['form'] = FormElement
     789
     790 -def submit_form(form, extra_values=None, open_http=None):
     791      """
     792      Helper function to submit a form.  Returns a file-like object, as from
     793      ``urllib.urlopen()``.  This object also has a ``.geturl()`` function,
     794      which shows the URL if there were any redirects.
     795
     796      You can use this like::
     797
     798          form = doc.forms[0]
     799          form.inputs['foo'].value = 'bar' # etc
     800          response = form.submit()
     801          doc = parse(response)
     802          doc.make_links_absolute(response.geturl())
     803
     804      To change the HTTP requester, pass a function as ``open_http`` keyword
     805      argument that opens the URL for you.  The function must have the following
     806      signature::
     807
     808          open_http(method, URL, values)
     809
     810      The action is one of 'GET' or 'POST', the URL is the target URL as a
     811      string, and the values are a sequence of ``(name, value)`` tuples with the
     812      form data.
     813      """
     814      values = form.form_values()
     815      if extra_values:
     816          if hasattr(extra_values, 'items'):
     817              extra_values = extra_values.items()
     818          values.extend(extra_values)
     819      if open_http is None:
     820          open_http = open_http_urllib
     821      return open_http(form.method, form.action, values)
     822
     823 -def open_http_urllib(method, url, values):
     824      ## FIXME: should test that it's not a relative URL or something
     825      try:
     826          from urllib import urlencode, urlopen
     827      except ImportError: # Python 3
     828          from urllib.request import urlopen
     829          from urllib.parse import urlencode
     830      if method == 'GET':
     831          if '?' in url:
     832              url += '&'
     833          else:
     834              url += '?'
     835          url += urlencode(values)
     836          data = None
     837      else:
     838          data = urlencode(values)
     839      return urlopen(url, data)
     840
     841 -class FieldsDict(DictMixin):
     842
     843 -    def __init__(self, inputs):
     844          self.inputs = inputs
     845 -    def __getitem__(self, item):
     846          return self.inputs[item].value
     847 -    def __setitem__(self, item, value):
     848          self.inputs[item].value = value
     849 -    def __delitem__(self, item):
     850          raise KeyError(
     851              "You cannot remove keys from ElementDict")
     852 -    def keys(self):
     853          return self.inputs.keys()
     854 -    def __contains__(self, item):
     855          return item in self.inputs
     856
     857 -    def __repr__(self):
     858          return '<%s for form %s>' % (
     859              self.__class__.__name__,
     860              self.inputs.form._name())
     861
     862 -class InputGetter(object):
     863
     864      """
     865      An accessor that represents all the input fields in a form.
     866
     867      You can get fields by name from this, with
     868      ``form.inputs['field_name']``.  If there are a set of checkboxes
     869      with the same name, they are returned as a list (a `CheckboxGroup`
     870      which also allows value setting).  Radio inputs are handled
     871      similarly.
     872
     873      You can also iterate over this to get all input elements.  This
     874      won't return the same thing as if you get all the names, as
     875      checkboxes and radio elements are returned individually.
     876      """
     877
     878      _name_xpath = etree.XPath(".//*[@name = $name and (local-name(.) = 'select' or local-name(.) = 'input' or local-name(.) = 'textarea')]")
     879      _all_xpath = etree.XPath(".//*[local-name() = 'select' or local-name() = 'input' or local-name() = 'textarea']")
     880
     881 -    def __init__(self, form):
     882          self.form = form
     883
     884 -    def __repr__(self):
     885          return '<%s for form %s>' % (
     886              self.__class__.__name__,
     887              self.form._name())
     888
     889      ## FIXME: there should be more methods, and it's unclear if this is
     890      ## a dictionary-like object or list-like object
     891
     892 -    def __getitem__(self, name):
     893          results = self._name_xpath(self.form, name=name)
     894          if results:
     895              type = results[0].get('type')
     896              if type == 'radio' and len(results) > 1:
     897                  group = RadioGroup(results)
     898                  group.name = name
     899                  return group
     900              elif type == 'checkbox' and len(results) > 1:
     901                  group = CheckboxGroup(results)
     902                  group.name = name
     903                  return group
     904              else:
     905                  # I don't like throwing away elements like this
     906                  return results[0]
     907          else:
     908              raise KeyError(
     909                  "No input element with the name %r" % name)
     910
     911 -    def __contains__(self, name):
     912          results = self._name_xpath(self.form, name=name)
     913          return bool(results)
     914
     915 -    def keys(self):
     916          names = set()
     917          for el in self:
     918              names.add(el.name)
     919          if None in names:
     920              names.remove(None)
     921          return list(names)
     922
     923 -    def __iter__(self):
     924          ## FIXME: kind of dumb to turn a list into an iterator, only
     925          ## to have it likely turned back into a list again :(
     926          return iter(self._all_xpath(self.form))
     927
     928 -class InputMixin(object):
     929
     930      """
     931      Mix-in for all input elements (input, select, and textarea)
     932      """
     933
     934
     935 -    def _name__get(self):
     936          """
     937          Get/set the name of the element
     938          """
     939          return self.get('name')
     940 -    def _name__set(self, value):
     941          self.set('name', value)
     942 -    def _name__del(self):
     943          if 'name' in self.attrib:
     944              del self.attrib['name']
     945      name = property(_name__get, _name__set, _name__del, doc=_name__get.__doc__)
     946
     947 -    def __repr__(self):
     948          type = getattr(self, 'type', None)
     949          if type:
     950              type = ' type=%r' % type
     951          else:
     952              type = ''
     953          return '<%s %x name=%r%s>' % (
     954              self.__class__.__name__, id(self), self.name, type)
     955
     956 -class TextareaElement(InputMixin, HtmlElement):
     957      """
     958      ``<textarea>`` element.  You can get the name with ``.name`` and
     959      get/set the value with ``.value``
     960      """
     961
     962 -    def _value__get(self):
     963          """
     964          Get/set the value (which is the contents of this element)
     965          """
     966          return self.text or ''
     967 -    def _value__set(self, value):
     968          self.text = value
     969 -    def _value__del(self):
     970          self.text = ''
     971      value = property(_value__get, _value__set, _value__del, doc=_value__get.__doc__)
     972
     973  HtmlElementClassLookup._default_element_classes['textarea'] = TextareaElement
     974
     975 -class SelectElement(InputMixin, HtmlElement):
     976      """
     977      ``<select>`` element.  You can get the name with ``.name``.
     978
     979      ``.value`` will be the value of the selected option, unless this
     980      is a multi-select element (``<select multiple>``), in which case
     981      it will be a set-like object.  In either case ``.value_options``
     982      gives the possible values.
     983
     984      The boolean attribute ``.multiple`` shows if this is a
     985      multi-select.
     986      """
     987
     988 -    def _value__get(self):
     989          """
     990          Get/set the value of this select (the selected option).
     991
     992          If this is a multi-select, this is a set-like object that
     993          represents all the selected options.
     994          """
     995          if self.multiple:
     996              return MultipleSelectOptions(self)
     997          for el in _options_xpath(self):
     998              if el.get('selected') is not None:
     999                  value = el.get('value')
    1000                  if value is None:
    1001                      value = el.text or ''
    1002                  if value:
    1003                      value = value.strip()
    1004                  return value
    1005          return None
    1006
    1007 -    def _value__set(self, value):
    1008          if self.multiple:
    1009              if isinstance(value, basestring):
    1010                  raise TypeError(
    1011                      "You must pass in a sequence")
    1012              self.value.clear()
    1013              self.value.update(value)
    1014              return
    1015          if value is not None:
    1016              value = value.strip()
    1017              for el in _options_xpath(self):
    1018                  opt_value = el.get('value')
    1019                  if opt_value is None:
    1020                      opt_value = el.text or ''
    1021                  if opt_value:
    1022                      opt_value = opt_value.strip()
    1023                  if opt_value == value:
    1024                      checked_option = el
    1025                      break
    1026              else:
    1027                  raise ValueError(
    1028                      "There is no option with the value of %r" % value)
    1029          for el in _options_xpath(self):
    1030              if 'selected' in el.attrib:
    1031                  del el.attrib['selected']
    1032          if value is not None:
    1033              checked_option.set('selected', '')
    1034
    1035 -    def _value__del(self):
    1036          # FIXME: should del be allowed at all?
    1037          if self.multiple:
    1038              self.value.clear()
    1039          else:
    1040              self.value = None
    1041
    1042      value = property(_value__get, _value__set, _value__del, doc=_value__get.__doc__)
    1043
    1044 -    def value_options(self):
    1045          """
    1046          All the possible values this select can have (the ``value``
    1047          attribute of all the ``<option>`` elements.
    1048          """
    1049          options = []
    1050          for el in _options_xpath(self):
    1051              value = el.get('value')
    1052              if value is None:
    1053                  value = el.text or ''
    1054              if value:
    1055                  value = value.strip()
    1056              options.append(value)
    1057          return options
    1058      value_options = property(value_options, doc=value_options.__doc__)
    1059
    1060 -    def _multiple__get(self):
    1061          """
    1062          Boolean attribute: is there a ``multiple`` attribute on this element.
    1063          """
    1064          return 'multiple' in self.attrib
    1065 -    def _multiple__set(self, value):
    1066          if value:
    1067              self.set('multiple', '')
    1068          elif 'multiple' in self.attrib:
    1069              del self.attrib['multiple']
    1070      multiple = property(_multiple__get, _multiple__set, doc=_multiple__get.__doc__)
    1071
    1072  HtmlElementClassLookup._default_element_classes['select'] = SelectElement
    1073
    1074 -class MultipleSelectOptions(SetMixin):
    1075      """
    1076      Represents all the selected options in a ``<select multiple>`` element.
    1077
    1078      You can add to this set-like option to select an option, or remove
    1079      to unselect the option.
    1080      """
    1081
    1082 -    def __init__(self, select):
    1083          self.select = select
    1084
    1085 -    def options(self):
    1086          """
    1087          Iterator of all the ``<option>`` elements.
    1088          """
    1089          return iter(_options_xpath(self.select))
    1090      options = property(options)
    1091
    1092 -    def __iter__(self):
    1093          for option in self.options:
    1094              yield option.get('value')
    1095
    1096 -    def add(self, item):
    1097          for option in self.options:
    1098              if option.get('value') == item:
    1099                  option.set('selected', '')
    1100                  break
    1101          else:
    1102              raise ValueError(
    1103                  "There is no option with the value %r" % item)
    1104
    1105 -    def remove(self, item):
    1106          for option in self.options:
    1107              if option.get('value') == item:
    1108                  if 'selected' in option.attrib:
    1109                      del option.attrib['selected']
    1110                  else:
    1111                      raise ValueError(
    1112                          "The option %r is not currently selected" % item)
    1113                  break
    1114          else:
    1115              raise ValueError(
    1116                  "There is not option with the value %r" % item)
    1117
    1118 -    def __repr__(self):
    1119          return '<%s {%s} for select name=%r>' % (
    1120              self.__class__.__name__,
    1121              ', '.join([repr(v) for v in self]),
    1122              self.select.name)
    1123
    1124 -class RadioGroup(list):
    1125      """
    1126      This object represents several ``<input type=radio>`` elements
    1127      that have the same name.
    1128
    1129      You can use this like a list, but also use the property
    1130      ``.value`` to check/uncheck inputs.  Also you can use
    1131      ``.value_options`` to get the possible values.
    1132      """
    1133
    1134 -    def _value__get(self):
    1135          """
    1136          Get/set the value, which checks the radio with that value (and
    1137          unchecks any other value).
    1138          """
    1139          for el in self:
    1140              if 'checked' in el.attrib:
    1141                  return el.get('value')
    1142          return None
    1143
    1144 -    def _value__set(self, value):
    1145          if value is not None:
    1146              for el in self:
    1147                  if el.get('value') == value:
    1148                      checked_option = el
    1149                      break
    1150              else:
    1151                  raise ValueError(
    1152                      "There is no radio input with the value %r" % value)
    1153          for el in self:
    1154              if 'checked' in el.attrib:
    1155                  del el.attrib['checked']
    1156          if value is not None:
    1157              checked_option.set('checked', '')
    1158
    1159 -    def _value__del(self):
    1160          self.value = None
    1161
    1162      value = property(_value__get, _value__set, _value__del, doc=_value__get.__doc__)
    1163
    1164 -    def value_options(self):
    1165          """
    1166          Returns a list of all the possible values.
    1167          """
    1168          return [el.get('value') for el in self]
    1169      value_options = property(value_options, doc=value_options.__doc__)
    1170
    1171 -    def __repr__(self):
    1172          return '%s(%s)' % (
    1173              self.__class__.__name__,
    1174              list.__repr__(self))
    1175
    1176 -class CheckboxGroup(list):
    1177      """
    1178      Represents a group of checkboxes (``<input type=checkbox>``) that
    1179      have the same name.
    1180
    1181      In addition to using this like a list, the ``.value`` attribute
    1182      returns a set-like object that you can add to or remove from to
    1183      check and uncheck checkboxes.  You can also use ``.value_options``
    1184      to get the possible values.
    1185      """
    1186
    1187 -    def _value__get(self):
    1188          """
    1189          Return a set-like object that can be modified to check or
    1190          uncheck individual checkboxes according to their value.
    1191          """
    1192          return CheckboxValues(self)
    1193 -    def _value__set(self, value):
    1194          self.value.clear()
    1195          if not hasattr(value, '__iter__'):
    1196              raise ValueError(
    1197                  "A CheckboxGroup (name=%r) must be set to a sequence (not %r)"
    1198                  % (self[0].name, value))
    1199          self.value.update(value)
    1200 -    def _value__del(self):
    1201          self.value.clear()
    1202      value = property(_value__get, _value__set, _value__del, doc=_value__get.__doc__)
    1203
    1204 -    def __repr__(self):
    1205          return '%s(%s)' % (
    1206              self.__class__.__name__, list.__repr__(self))
    1207
    1208 -class CheckboxValues(SetMixin):
    1209
    1210      """
    1211      Represents the values of the checked checkboxes in a group of
    1212      checkboxes with the same name.
    1213      """
    1214
    1215 -    def __init__(self, group):
    1216          self.group = group
    1217
    1218 -    def __iter__(self):
    1219          return iter([
    1220              el.get('value')
    1221              for el in self.group
    1222              if 'checked' in el.attrib])
    1223
    1224 -    def add(self, value):
    1225          for el in self.group:
    1226              if el.get('value') == value:
    1227                  el.set('checked', '')
    1228                  break
    1229          else:
    1230              raise KeyError("No checkbox with value %r" % value)
    1231
    1232 -    def remove(self, value):
    1233          for el in self.group:
    1234              if el.get('value') == value:
    1235                  if 'checked' in el.attrib:
    1236                      del el.attrib['checked']
    1237                  else:
    1238                      raise KeyError(
    1239                          "The checkbox with value %r was already unchecked" % value)
    1240                  break
    1241          else:
    1242              raise KeyError(
    1243                  "No checkbox with value %r" % value)
    1244
    1245 -    def __repr__(self):
    1246          return '<%s {%s} for checkboxes name=%r>' % (
    1247              self.__class__.__name__,
    1248              ', '.join([repr(v) for v in self]),
    1249              self.group.name)
    1250
    1251 -class InputElement(InputMixin, HtmlElement):
    1252      """
    1253      Represents an ``<input>`` element.
    1254
    1255      You can get the type with ``.type`` (which is lower-cased and
    1256      defaults to ``'text'``).
    1257
    1258      Also you can get and set the value with ``.value``
    1259
    1260      Checkboxes and radios have the attribute ``input.checkable ==
    1261      True`` (for all others it is false) and a boolean attribute
    1262      ``.checked``.
    1263
    1264      """
    1265
    1266      ## FIXME: I'm a little uncomfortable with the use of .checked
    1267 -    def _value__get(self):
    1268          """
    1269          Get/set the value of this element, using the ``value`` attribute.
    1270
    1271          Also, if this is a checkbox and it has no value, this defaults
    1272          to ``'on'``.  If it is a checkbox or radio that is not
    1273          checked, this returns None.
    1274          """
    1275          if self.checkable:
    1276              if self.checked:
    1277                  return self.get('value') or 'on'
    1278              else:
    1279                  return None
    1280          return self.get('value')
    1281 -    def _value__set(self, value):
    1282          if self.checkable:
    1283              if not value:
    1284                  self.checked = False
    1285              else:
    1286                  self.checked = True
    1287                  if isinstance(value, basestring):
    1288                      self.set('value', value)
    1289          else:
    1290              self.set('value', value)
    1291 -    def _value__del(self):
    1292          if self.checkable:
    1293              self.checked = False
    1294          else:
    1295              if 'value' in self.attrib:
    1296                  del self.attrib['value']
    1297      value = property(_value__get, _value__set, _value__del, doc=_value__get.__doc__)
    1298
    1299 -    def _type__get(self):
    1300          """
    1301          Return the type of this element (using the type attribute).
    1302          """
    1303          return self.get('type', 'text').lower()
    1304 -    def _type__set(self, value):
    1305          self.set('type', value)
    1306      type = property(_type__get, _type__set, doc=_type__get.__doc__)
    1307
    1308 -    def checkable(self):
    1309          """
    1310          Boolean: can this element be checked?
    1311          """
    1312          return self.type in ['checkbox', 'radio']
    1313      checkable = property(checkable, doc=checkable.__doc__)
    1314
    1315 -    def _checked__get(self):
    1316          """
    1317          Boolean attribute to get/set the presence of the ``checked``
    1318          attribute.
    1319
    1320          You can only use this on checkable input types.
    1321          """
    1322          if not self.checkable:
    1323              raise AttributeError('Not a checkable input type')
    1324          return 'checked' in self.attrib
    1325 -    def _checked__set(self, value):
    1326          if not self.checkable:
    1327              raise AttributeError('Not a checkable input type')
    1328          if value:
    1329              self.set('checked', '')
    1330          else:
    1331              if 'checked' in self.attrib:
    1332                  del self.attrib['checked']
    1333      checked = property(_checked__get, _checked__set, doc=_checked__get.__doc__)
    1334
    1335  HtmlElementClassLookup._default_element_classes['input'] = InputElement
    1336
    1337 -class LabelElement(HtmlElement):
    1338      """
    1339      Represents a ``<label>`` element.
    1340
    1341      Label elements are linked to other elements with their ``for``
    1342      attribute.  You can access this element with ``label.for_element``.
    1343      """
    1344
    1345 -    def _for_element__get(self):
    1346          """
    1347          Get/set the element this label points to.  Return None if it
    1348          can't be found.
    1349          """
    1350          id = self.get('for')
    1351          if not id:
    1352              return None
    1353          return self.body.get_element_by_id(id)
    1354 -    def _for_element__set(self, other):
    1355          id = other.get('id')
    1356          if not id:
    1357              raise TypeError(
    1358                  "Element %r has no id attribute" % other)
    1359          self.set('for', id)
    1360 -    def _for_element__del(self):
    1361          if 'id' in self.attrib:
    1362              del self.attrib['id']
    1363      for_element = property(_for_element__get, _for_element__set, _for_element__del,
    1364                             doc=_for_element__get.__doc__)
    1365
    1366  HtmlElementClassLookup._default_element_classes['label'] = LabelElement
    1367
    1368  ############################################################
    1369  ## Serialization
    1370  ############################################################
    1371
    1372 -def html_to_xhtml(html):
    1373      """Convert all tags in an HTML tree to XHTML by moving them to the
    1374      XHTML namespace.
    1375      """
    1376      try:
    1377          html = html.getroot()
    1378      except AttributeError:
    1379          pass
    1380      prefix = "{%s}" % XHTML_NAMESPACE
    1381      for el in html.iter():
    1382          tag = el.tag
    1383          if isinstance(tag, basestring):
    1384              if tag[0] != '{':
    1385                  el.tag = prefix + tag
    1386
    1387 -def xhtml_to_html(xhtml):
    1388      """Convert all tags in an XHTML tree to HTML by removing their
    1389      XHTML namespace.
    1390      """
    1391      try:
    1392          xhtml = xhtml.getroot()
    1393      except AttributeError:
    1394          pass
    1395      prefix = "{%s}" % XHTML_NAMESPACE
    1396      prefix_len = len(prefix)
    1397      for el in xhtml.iter(prefix + "*"):
    1398          el.tag = el.tag[prefix_len:]
    1399
    1400  # This isn't a general match, but it's a match for what libxml2
    1401  # specifically serialises:
    1402  __str_replace_meta_content_type = re.compile(
    1403      r'<meta http-equiv="Content-Type"[^>]*>').sub
    1404  __bytes_replace_meta_content_type = re.compile(
    1405      r'<meta http-equiv="Content-Type"[^>]*>'.encode('ASCII')).sub
    1406
    1407 -def tostring(doc, pretty_print=False, include_meta_content_type=False,
    1408               encoding=None, method="html"):
    1409      """Return an HTML string representation of the document.
    1410
    1411      Note: if include_meta_content_type is true this will create a
    1412      ``<meta http-equiv="Content-Type" ...>`` tag in the head;
    1413      regardless of the value of include_meta_content_type any existing
    1414      ``<meta http-equiv="Content-Type" ...>`` tag will be removed
    1415
    1416      The ``encoding`` argument controls the output encoding (defauts to
    1417      ASCII, with &#...; character references for any characters outside
    1418      of ASCII).
    1419
    1420      The ``method`` argument defines the output method.  It defaults to
    1421      'html', but can also be 'xml' for xhtml output, or 'text' to
    1422      serialise to plain text without markup.  Note that you can pass
    1423      the builtin ``unicode`` type as ``encoding`` argument to serialise
    1424      to a unicode string.
    1425
    1426      Example::
    1427
    1428          >>> from lxml import html
    1429          >>> root = html.fragment_fromstring('<p>Hello<br>world!</p>')
    1430
    1431          >>> html.tostring(root)
    1432          b'<p>Hello<br>world!</p>'
    1433          >>> html.tostring(root, method='html')
    1434          b'<p>Hello<br>world!</p>'
    1435
    1436          >>> html.tostring(root, method='xml')
    1437          b'<p>Hello<br/>world!</p>'
    1438
    1439          >>> html.tostring(root, method='text')
    1440          b'Helloworld!'
    1441
    1442          >>> html.tostring(root, method='text', encoding=unicode)
    1443          u'Helloworld!'
    1444      """
    1445      html = etree.tostring(doc, method=method, pretty_print=pretty_print,
    1446                            encoding=encoding)
    1447      if not include_meta_content_type:
    1448          if isinstance(html, str):
    1449              html = __str_replace_meta_content_type('', html)
    1450          else:
    1451              html = __bytes_replace_meta_content_type(bytes(), html)
    1452      return html
    1453
    1454  tostring.__doc__ = __fix_docstring(tostring.__doc__)
    1455
    1456 -def open_in_browser(doc):
    1457      """
    1458      Open the HTML document in a web browser (saving it to a temporary
    1459      file to open it).
    1460      """
    1461      import os
    1462      import webbrowser
    1463      try:
    1464          write_doc = doc.write
    1465      except AttributeError:
    1466          write_doc = etree.ElementTree(element=doc).write
    1467      fn = os.tempnam() + '.html'
    1468      write_doc(fn, method="html")
    1469      url = 'file://' + fn.replace(os.path.sep, '/')
    1470      print(url)
    1471      webbrowser.open(url)
    1472
    1473  ################################################################################
    1474  # configure Element class lookup
    1475  ################################################################################
    1476
    1477 -class HTMLParser(etree.HTMLParser):
    1478 -    def __init__(self, **kwargs):
    1479          super(HTMLParser, self).__init__(**kwargs)
    1480          self.set_element_class_lookup(HtmlElementClassLookup())
    1481
    1482 -class XHTMLParser(etree.XMLParser):
    1483 -    def __init__(self, **kwargs):
    1484          super(XHTMLParser, self).__init__(**kwargs)
    1485          self.set_element_class_lookup(HtmlElementClassLookup())
    1486
    1487 -def Element(*args, **kw):
    1488      """Create a new HTML Element.
    1489
    1490      This can also be used for XHTML documents.
    1491      """
    1492      v = html_parser.makeelement(*args, **kw)
    1493      return v
    1494
    1495  html_parser = HTMLParser()
    1496  xhtml_parser = XHTMLParser()
    1497