"""Shared support for scanning document type declarations in HTML and XHTML.
This module is used as a foundation for the html.parser module. It has no
documented public API and should not be used directly.
"""
import re
_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match
_declstringlit_match = re.compile(r'(\'[^\']*\'|"[^"]*")\s*').match
_commentclose = re.compile(r'--\s*>')
_markedsectionclose = re.compile(r']\s*]\s*>')
# An analysis of the MS-Word extensions is available at
# http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
_msmarkedsectionclose = re.compile(r']\s*>')
del re
class ParserBase:
"""Parser base class which provides some common support methods used
by the SGML/HTML and XHTML parsers."""
def __init__(self):
if self.__class__ is ParserBase:
raise RuntimeError(
"_markupbase.ParserBase must be subclassed")
def error(self, message):
raise NotImplementedError(
"subclasses of ParserBase must override error()")
def reset(self):
self.lineno = 1
self.offset = 0
def getpos(self):
"""Return current line number and offset."""
return self.lineno, self.offset
# Internal -- update line number and offset. This should be
# called for each piece of data exactly once, in order -- in other
# words the concatenation of all the input strings to this
# function should be exactly the entire input.
def updatepos(self, i, j):
if i >= j:
return j
rawdata = self.rawdata
nlines = rawdata.count("\n", i, j)
if nlines:
self.lineno = self.lineno + nlines
pos = rawdata.rindex("\n", i, j) # Should not fail
self.offset = j-(pos+1)
else:
self.offset = self.offset + j-i
return j
_decl_otherchars = ''
# Internal -- parse declaration (for use by subclasses).
def parse_declaration(self, i):
# This is some sort of declaration; in "HTML as
# deployed," this should only be the document type
# declaration ("<!DOCTYPE html...>").
# ISO 8879:1986, however, has more complex
# declaration syntax for elements in <!...>, including:
# --comment--
# [marked section]
# name in the following list: ENTITY, DOCTYPE, ELEMENT,
# ATTLIST, NOTATION, SHORTREF, USEMAP,
# LINKTYPE, LINK, IDLINK, USELINK, SYSTEM
rawdata = self.rawdata
j = i + 2
assert rawdata[i:j] == "<!", "unexpected call to parse_declaration"
if rawdata[j:j+1] == ">":
# the empty comment <!>
return j + 1
if rawdata[j:j+1] in ("-", ""):
# Start of comment followed by buffer boundary,
# or just a buffer boundary.
return -1
# A simple, practical version could look like: ((name|stringlit) S*) + '>'
n = len(rawdata)
if rawdata[j:j+2] == '--': #comment
# Locate --.*-- as the body of the comment
return self.parse_comment(i)
elif rawdata[j] == '[': #marked section
# Locate [statusWord [...arbitrary SGML...]] as the body of the marked section
# Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA
# Note that this is extended by Microsoft Office "Save as Web" function
# to include [if...] and [endif].
return self.parse_marked_section(i)
else: #all other declaration elements
decltype, j = self._scan_name(j, i)
if j < 0:
return j
if decltype == "doctype":
self._decl_otherchars = ''
while j < n:
c = rawdata[j]
if c == ">":
# end of declaration syntax
data = rawdata[i+2:j]
if decltype == "doctype":
self.handle_decl(data)
else:
# According to the HTML5 specs sections "8.2.4.44 Bogus
# comment state" and "8.2.4.45 Markup declaration open
# state", a comment token should be emitted.
# Calling unknown_decl provides more flexibility though.
self.unknown_decl(data)
return j + 1
if c in "\"'":
m = _declstringlit_match(rawdata, j)
if not m:
return -1 # incomplete
j = m.end()
elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
name, j = self._scan_name(j, i)
elif c in self._decl_otherchars:
j = j + 1
elif c == "[":
# this could be handled in a separate doctype parser
if decltype == "doctype":
j = self._parse_doctype_subset(j + 1, i)
elif decltype in {"attlist", "linktype", "link", "element"}:
# must tolerate []'d groups in a content model in an element declaration
# also in data attribute specifications of attlist declaration
# also link type declaration subsets in linktype declarations
# also link attribute specification lists in link declarations
self.error("unsupported '[' char in %s declaration" % decltype)
else:
self.error("unexpected '[' char in declaration")
else:
self.error(
"unexpected %r char in declaration" % rawdata[j])
if j < 0:
return j
return -1 # incomplete
# Internal -- parse a marked section
# Override this to handle MS-word extension syntax <![if word]>content<![endif]>
def parse_marked_section(self, i, report=1):
rawdata= self.rawdata
assert rawdata[i:i+3] == '<![', "unexpected call to parse_marked_section()"
sectName, j = self._scan_name( i+3, i )
if j < 0:
return j
if sectName in {"temp", "cdata", "ignore", "include", "rcdata"}:
# look for standard ]]> ending
match= _markedsectionclose.search(rawdata, i+3)
elif sectName in {"if", "else", "endif"}:
# look for MS Office ]> ending
match= _msmarkedsectionclose.search(rawdata, i+3)
else:
self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
if not match:
return -1
if report:
j = match.start(0)
self.unknown_decl(rawdata[i+3: j])
return match.end(0)
# Internal -- parse comment, return length or -1 if not terminated
def parse_comment(self, i, report=1):
rawdata = self.rawdata
if rawdata[i:i+4] != '<!--':
self.error('unexpected call to parse_comment()')
match = _commentclose.search(rawdata, i+4)
if not match:
return -1
if report:
j = match.start(0)
self.handle_comment(rawdata[i+4: j])
return match.end(0)
# Internal -- scan past the internal subset in a <!DOCTYPE declaration,
# returning the index just past any whitespace following the trailing ']'.
def _parse_doctype_subset(self, i, declstartpos):
rawdata = self.rawdata
n = len(rawdata)
j = i
while j < n:
c = rawdata[j]
if c == "<":
s = rawdata[j:j+2]
if s == "<":
# end of buffer; incomplete
return -1
if s != "<!":
self.updatepos(declstartpos, j + 1)
self.error("unexpected char in internal subset (in %r)" % s)
if (j + 2) == n:
# end of buffer; incomplete
return -1
if (j + 4) > n:
# end of buffer; incomplete
return -1
if rawdata[j:j+4] == "<!--":
j = self.parse_comment(j, report=0)
if j < 0:
return j
continue
name, j = self._scan_name(j + 2, declstartpos)
if j == -1:
return -1
if name not in {"attlist", "element", "entity", "notation"}:
self.updatepos(declstartpos, j + 2)
self.error(
"unknown declaration %r in internal subset" % name)
# handle the individual names
meth = getattr(self, "_parse_doctype_" + name)
j = meth(j, declstartpos)
if j < 0:
return j
elif c == "%":
# parameter entity reference
if (j + 1) == n:
# end of buffer; incomplete
return -1
s, j = self._scan_name(j + 1, declstartpos)
if j < 0:
return j
if rawdata[j] == ";":
j = j + 1
elif c == "]":
j = j + 1
while j < n and rawdata[j].isspace():
j = j + 1
if j < n:
if rawdata[j] == ">":
return j
self.updatepos(declstartpos, j)
self.error("unexpected char after internal subset")
else:
return -1
elif c.isspace():
j = j + 1
else:
self.updatepos(declstartpos, j)
self.error("unexpected char %r in internal subset" % c)
# end of buffer reached
return -1
# Internal -- scan past <!ELEMENT declarations
def _parse_doctype_element(self, i, declstartpos):
name, j = self._scan_name(i, declstartpos)
if j == -1:
return -1
# style content model; just skip until '>'
rawdata = self.rawdata
if '>' in rawdata[j:]:
return rawdata.find(">", j) + 1
return -1
# Internal -- scan past <!ATTLIST declarations
def _parse_doctype_attlist(self, i, declstartpos):
rawdata = self.rawdata
name, j = self._scan_name(i, declstartpos)
c = rawdata[j:j+1]
if c == "":
return -1
if c == ">":
return j + 1
while 1:
# scan a series of attribute descriptions; simplified:
# name type [value] [#constraint]
name, j = self._scan_name(j, declstartpos)
if j < 0:
return j
c = rawdata[j:j+1]
if c == "":
return -1
if c == "(":
# an enumerated type; look for ')'
if ")" in rawdata[j:]:
j = rawdata.find(")", j) + 1
else:
return -1
while rawdata[j:j+1].isspace():
j = j + 1
if not rawdata[j:]:
# end of buffer, incomplete
return -1
else:
name, j = self._scan_name(j, declstartpos)
c = rawdata[j:j+1]
if not c:
return -1
if c in "'\"":
m = _declstringlit_match(rawdata, j)
if m:
j = m.end()
else:
return -1
c = rawdata[j:j+1]
if not c:
return -1
if c == "#":
if rawdata[j:] == "#":
# end of buffer
return -1
name, j = self._scan_name(j + 1, declstartpos)
if j < 0:
return j
c = rawdata[j:j+1]
if not c:
return -1
if c == '>':
# all done
return j + 1
# Internal -- scan past <!NOTATION declarations
def _parse_doctype_notation(self, i, declstartpos):
name, j = self._scan_name(i, declstartpos)
if j < 0:
return j
rawdata = self.rawdata
while 1:
c = rawdata[j:j+1]
if not c:
# end of buffer; incomplete
return -1
if c == '>':
return j + 1
if c in "'\"":
m = _declstringlit_match(rawdata, j)
if not m:
return -1
j = m.end()
else:
name, j = self._scan_name(j, declstartpos)
if j < 0:
return j
# Internal -- scan past <!ENTITY declarations
def _parse_doctype_entity(self, i, declstartpos):
rawdata = self.rawdata
if rawdata[i:i+1] == "%":
j = i + 1
while 1:
c = rawdata[j:j+1]
if not c:
return -1
if c.isspace():
j = j + 1
else:
break
else:
j = i
name, j = self._scan_name(j, declstartpos)
if j < 0:
return j
while 1:
c = self.rawdata[j:j+1]
if not c:
return -1
if c in "'\"":
m = _declstringlit_match(rawdata, j)
if m:
j = m.end()
else:
return -1 # incomplete
elif c == ">":
return j + 1
else:
name, j = self._scan_name(j, declstartpos)
if j < 0:
return j
# Internal -- scan a name token and the new position and the token, or
# return -1 if we've reached the end of the buffer.
def _scan_name(self, i, declstartpos):
rawdata = self.rawdata
n = len(rawdata)
if i == n:
return None, -1
m = _declname_match(rawdata, i)
if m:
s = m.group()
name = s.strip()
if (i + len(s)) == n:
return None, -1 # end of buffer
return name.lower(), m.end()
else:
self.updatepos(declstartpos, i)
self.error("expected name token at %r"
% rawdata[declstartpos:declstartpos+20])
# To be overridden -- handlers for unknown objects
def unknown_decl(self, data):
pass
| Name | Type | Size | Permission | Actions |
|---|---|---|---|---|
| __pycache__ | Folder | 0755 |
|
|
| asyncio | Folder | 0755 |
|
|
| collections | Folder | 0755 |
|
|
| concurrent | Folder | 0755 |
|
|
| config-3.6m-x86_64-linux-gnu | Folder | 0755 |
|
|
| ctypes | Folder | 0755 |
|
|
| curses | Folder | 0755 |
|
|
| dbm | Folder | 0755 |
|
|
| distutils | Folder | 0755 |
|
|
| Folder | 0755 |
|
||
| encodings | Folder | 0755 |
|
|
| ensurepip | Folder | 0755 |
|
|
| html | Folder | 0755 |
|
|
| http | Folder | 0755 |
|
|
| importlib | Folder | 0755 |
|
|
| json | Folder | 0755 |
|
|
| lib-dynload | Folder | 0755 |
|
|
| lib2to3 | Folder | 0755 |
|
|
| logging | Folder | 0755 |
|
|
| multiprocessing | Folder | 0755 |
|
|
| pydoc_data | Folder | 0755 |
|
|
| site-packages | Folder | 0755 |
|
|
| sqlite3 | Folder | 0755 |
|
|
| test | Folder | 0755 |
|
|
| unittest | Folder | 0755 |
|
|
| urllib | Folder | 0755 |
|
|
| venv | Folder | 0755 |
|
|
| wsgiref | Folder | 0755 |
|
|
| xml | Folder | 0755 |
|
|
| xmlrpc | Folder | 0755 |
|
|
| __future__.py | File | 4.73 KB | 0644 |
|
| __phello__.foo.py | File | 64 B | 0644 |
|
| _bootlocale.py | File | 1.27 KB | 0644 |
|
| _collections_abc.py | File | 25.77 KB | 0644 |
|
| _compat_pickle.py | File | 8.54 KB | 0644 |
|
| _compression.py | File | 5.21 KB | 0644 |
|
| _dummy_thread.py | File | 5 KB | 0644 |
|
| _markupbase.py | File | 14.26 KB | 0644 |
|
| _osx_support.py | File | 18.69 KB | 0644 |
|
| _pydecimal.py | File | 224.83 KB | 0644 |
|
| _pyio.py | File | 86.03 KB | 0644 |
|
| _sitebuiltins.py | File | 3.04 KB | 0644 |
|
| _strptime.py | File | 24.17 KB | 0644 |
|
| _sysconfigdata_dm_linux_x86_64-linux-gnu.py | File | 29.48 KB | 0644 |
|
| _sysconfigdata_m_linux_x86_64-linux-gnu.py | File | 29.66 KB | 0644 |
|
| _threading_local.py | File | 7.04 KB | 0644 |
|
| _weakrefset.py | File | 5.57 KB | 0644 |
|
| abc.py | File | 8.52 KB | 0644 |
|
| aifc.py | File | 31.69 KB | 0644 |
|
| antigravity.py | File | 477 B | 0644 |
|
| argparse.py | File | 88.25 KB | 0644 |
|
| ast.py | File | 11.88 KB | 0644 |
|
| asynchat.py | File | 11.06 KB | 0644 |
|
| asyncore.py | File | 19.69 KB | 0644 |
|
| base64.py | File | 19.91 KB | 0755 |
|
| bdb.py | File | 23 KB | 0644 |
|
| binhex.py | File | 13.63 KB | 0644 |
|
| bisect.py | File | 2.53 KB | 0644 |
|
| bz2.py | File | 12.19 KB | 0644 |
|
| cProfile.py | File | 5.25 KB | 0755 |
|
| calendar.py | File | 22.67 KB | 0644 |
|
| cgi.py | File | 36.35 KB | 0755 |
|
| cgitb.py | File | 11.74 KB | 0644 |
|
| chunk.py | File | 5.3 KB | 0644 |
|
| cmd.py | File | 14.51 KB | 0644 |
|
| code.py | File | 10.37 KB | 0644 |
|
| codecs.py | File | 35.43 KB | 0644 |
|
| codeop.py | File | 5.85 KB | 0644 |
|
| colorsys.py | File | 3.97 KB | 0644 |
|
| compileall.py | File | 11.84 KB | 0644 |
|
| configparser.py | File | 52.34 KB | 0644 |
|
| contextlib.py | File | 12.85 KB | 0644 |
|
| copy.py | File | 8.61 KB | 0644 |
|
| copyreg.py | File | 6.84 KB | 0644 |
|
| crypt.py | File | 1.82 KB | 0644 |
|
| csv.py | File | 15.8 KB | 0644 |
|
| datetime.py | File | 80.11 KB | 0644 |
|
| decimal.py | File | 320 B | 0644 |
|
| difflib.py | File | 82.4 KB | 0644 |
|
| dis.py | File | 17.71 KB | 0644 |
|
| doctest.py | File | 101.94 KB | 0644 |
|
| dummy_threading.py | File | 2.75 KB | 0644 |
|
| enum.py | File | 32.82 KB | 0644 |
|
| filecmp.py | File | 9.6 KB | 0644 |
|
| fileinput.py | File | 14.13 KB | 0644 |
|
| fnmatch.py | File | 3.09 KB | 0644 |
|
| formatter.py | File | 14.79 KB | 0644 |
|
| fractions.py | File | 23.08 KB | 0644 |
|
| ftplib.py | File | 34.78 KB | 0644 |
|
| functools.py | File | 30.61 KB | 0644 |
|
| genericpath.py | File | 4.64 KB | 0644 |
|
| getopt.py | File | 7.31 KB | 0644 |
|
| getpass.py | File | 5.85 KB | 0644 |
|
| gettext.py | File | 21.03 KB | 0644 |
|
| glob.py | File | 5.51 KB | 0644 |
|
| gzip.py | File | 19.86 KB | 0644 |
|
| hashlib.py | File | 8.59 KB | 0644 |
|
| heapq.py | File | 22.39 KB | 0644 |
|
| hmac.py | File | 6.23 KB | 0644 |
|
| imaplib.py | File | 52.05 KB | 0644 |
|
| imghdr.py | File | 3.71 KB | 0644 |
|
| imp.py | File | 10.42 KB | 0644 |
|
| inspect.py | File | 114.22 KB | 0644 |
|
| io.py | File | 3.43 KB | 0644 |
|
| ipaddress.py | File | 75.99 KB | 0644 |
|
| keyword.py | File | 2.17 KB | 0755 |
|
| linecache.py | File | 5.19 KB | 0644 |
|
| locale.py | File | 75.49 KB | 0644 |
|
| lzma.py | File | 12.68 KB | 0644 |
|
| macpath.py | File | 5.83 KB | 0644 |
|
| macurl2path.py | File | 2.67 KB | 0644 |
|
| mailbox.py | File | 76.78 KB | 0644 |
|
| mailcap.py | File | 8.85 KB | 0644 |
|
| mimetypes.py | File | 20.55 KB | 0644 |
|
| modulefinder.py | File | 22.49 KB | 0644 |
|
| netrc.py | File | 5.55 KB | 0644 |
|
| nntplib.py | File | 42.07 KB | 0644 |
|
| ntpath.py | File | 22.55 KB | 0644 |
|
| nturl2path.py | File | 2.39 KB | 0644 |
|
| numbers.py | File | 10 KB | 0644 |
|
| opcode.py | File | 5.69 KB | 0644 |
|
| operator.py | File | 10.61 KB | 0644 |
|
| optparse.py | File | 58.96 KB | 0644 |
|
| os.py | File | 36.65 KB | 0644 |
|
| pathlib.py | File | 47.83 KB | 0644 |
|
| pdb.py | File | 59.88 KB | 0755 |
|
| pickle.py | File | 54.39 KB | 0644 |
|
| pickletools.py | File | 89.62 KB | 0644 |
|
| pipes.py | File | 8.71 KB | 0644 |
|
| pkgutil.py | File | 20.82 KB | 0644 |
|
| platform.py | File | 46.11 KB | 0755 |
|
| plistlib.py | File | 31.53 KB | 0644 |
|
| poplib.py | File | 14.61 KB | 0644 |
|
| posixpath.py | File | 15.4 KB | 0644 |
|
| pprint.py | File | 20.37 KB | 0644 |
|
| profile.py | File | 21.51 KB | 0755 |
|
| pstats.py | File | 25.94 KB | 0644 |
|
| pty.py | File | 4.65 KB | 0644 |
|
| py_compile.py | File | 7.01 KB | 0644 |
|
| pyclbr.py | File | 13.24 KB | 0644 |
|
| pydoc.py | File | 101.08 KB | 0644 |
|
| queue.py | File | 8.57 KB | 0644 |
|
| quopri.py | File | 7.09 KB | 0755 |
|
| random.py | File | 26.8 KB | 0644 |
|
| re.py | File | 15.19 KB | 0644 |
|
| reprlib.py | File | 5.21 KB | 0644 |
|
| rlcompleter.py | File | 6.93 KB | 0644 |
|
| runpy.py | File | 11.68 KB | 0644 |
|
| sched.py | File | 6.36 KB | 0644 |
|
| secrets.py | File | 1.99 KB | 0644 |
|
| selectors.py | File | 18.98 KB | 0644 |
|
| shelve.py | File | 8.32 KB | 0644 |
|
| shlex.py | File | 12.65 KB | 0644 |
|
| shutil.py | File | 39.87 KB | 0644 |
|
| signal.py | File | 2.07 KB | 0644 |
|
| site.py | File | 20.77 KB | 0644 |
|
| smtpd.py | File | 33.91 KB | 0755 |
|
| smtplib.py | File | 43.18 KB | 0755 |
|
| sndhdr.py | File | 6.92 KB | 0644 |
|
| socket.py | File | 26.8 KB | 0644 |
|
| socketserver.py | File | 26.38 KB | 0644 |
|
| sre_compile.py | File | 18.88 KB | 0644 |
|
| sre_constants.py | File | 6.66 KB | 0644 |
|
| sre_parse.py | File | 35.68 KB | 0644 |
|
| ssl.py | File | 43.47 KB | 0644 |
|
| stat.py | File | 4.92 KB | 0644 |
|
| statistics.py | File | 20.19 KB | 0644 |
|
| string.py | File | 11.52 KB | 0644 |
|
| stringprep.py | File | 12.61 KB | 0644 |
|
| struct.py | File | 257 B | 0644 |
|
| subprocess.py | File | 60.88 KB | 0644 |
|
| sunau.py | File | 17.67 KB | 0644 |
|
| symbol.py | File | 2.07 KB | 0755 |
|
| symtable.py | File | 7.11 KB | 0644 |
|
| sysconfig.py | File | 24.29 KB | 0644 |
|
| tabnanny.py | File | 11.14 KB | 0755 |
|
| tarfile.py | File | 104.88 KB | 0755 |
|
| telnetlib.py | File | 22.59 KB | 0644 |
|
| tempfile.py | File | 27.41 KB | 0644 |
|
| textwrap.py | File | 19.1 KB | 0644 |
|
| this.py | File | 1003 B | 0644 |
|
| threading.py | File | 48.96 KB | 0644 |
|
| timeit.py | File | 13.03 KB | 0755 |
|
| token.py | File | 3 KB | 0644 |
|
| tokenize.py | File | 28.8 KB | 0644 |
|
| trace.py | File | 28.06 KB | 0755 |
|
| traceback.py | File | 22.91 KB | 0644 |
|
| tracemalloc.py | File | 16.27 KB | 0644 |
|
| tty.py | File | 879 B | 0644 |
|
| types.py | File | 8.66 KB | 0644 |
|
| typing.py | File | 78.39 KB | 0644 |
|
| uu.py | File | 6.6 KB | 0755 |
|
| uuid.py | File | 23.46 KB | 0644 |
|
| warnings.py | File | 18.05 KB | 0644 |
|
| wave.py | File | 17.29 KB | 0644 |
|
| weakref.py | File | 19.99 KB | 0644 |
|
| webbrowser.py | File | 21.26 KB | 0755 |
|
| xdrlib.py | File | 5.77 KB | 0644 |
|
| zipapp.py | File | 6.99 KB | 0644 |
|
| zipfile.py | File | 78.05 KB | 0644 |
|