Showing the changes to sys/lib/python/HTMLParser.py (1 of 3706 files). Show all
Unified · Split
| @@ -0,0 +1,369 @@ | |||
| 1 | """A parser for HTML and XHTML.""" | ||
| 2 | |||
| 3 | # This file is based on sgmllib.py, but the API is slightly different. | ||
| 4 | |||
| 5 | # XXX There should be a way to distinguish between PCDATA (parsed | ||
| 6 | # character data -- the normal case), RCDATA (replaceable character | ||
| 7 | # data -- only char and entity references and end tags are special) | ||
| 8 | # and CDATA (character data -- only end tags are special). | ||
| 9 | |||
| 10 | |||
| 11 | import markupbase | ||
| 12 | import re | ||
| 13 | |||
| 14 | # Regular expressions used for parsing | ||
| 15 | |||
| 16 | interesting_normal = re.compile('[&<]') | ||
| 17 | interesting_cdata = re.compile(r'<(/|\Z)') | ||
| 18 | incomplete = re.compile('&[a-zA-Z#]') | ||
| 19 | |||
| 20 | entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') | ||
| 21 | charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') | ||
| 22 | |||
| 23 | starttagopen = re.compile('<[a-zA-Z]') | ||
| 24 | piclose = re.compile('>') | ||
| 25 | commentclose = re.compile(r'--\s*>') | ||
| 26 | tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*') | ||
| 27 | attrfind = re.compile( | ||
| 28 | r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*' | ||
| 29 | r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~@]*))?') | ||
| 30 | |||
| 31 | locatestarttagend = re.compile(r""" | ||
| 32 | <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name | ||
| 33 | (?:\s+ # whitespace before attribute name | ||
| 34 | (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name | ||
| 35 | (?:\s*=\s* # value indicator | ||
| 36 | (?:'[^']*' # LITA-enclosed value | ||
| 37 | |\"[^\"]*\" # LIT-enclosed value | ||
| 38 | |[^'\">\s]+ # bare value | ||
| 39 | ) | ||
| 40 | )? | ||
| 41 | ) | ||
| 42 | )* | ||
| 43 | \s* # trailing whitespace | ||
| 44 | """, re.VERBOSE) | ||
| 45 | endendtag = re.compile('>') | ||
| 46 | endtagfind = re.compile('</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>') | ||
| 47 | |||
| 48 | |||
| 49 | class HTMLParseError(Exception): | ||
| 50 | """Exception raised for all parse errors.""" | ||
| 51 | |||
| 52 | def __init__(self, msg, position=(None, None)): | ||
| 53 | assert msg | ||
| 54 | self.msg = msg | ||
| 55 | self.lineno = position[0] | ||
| 56 | self.offset = position[1] | ||
| 57 | |||
| 58 | def __str__(self): | ||
| 59 | result = self.msg | ||
| 60 | if self.lineno is not None: | ||
| 61 | result = result + ", at line %d" % self.lineno | ||
| 62 | if self.offset is not None: | ||
| 63 | result = result + ", column %d" % (self.offset + 1) | ||
| 64 | return result | ||
| 65 | |||
| 66 | |||
| 67 | class HTMLParser(markupbase.ParserBase): | ||
| 68 | """Find tags and other markup and call handler functions. | ||
| 69 | |||
| 70 | Usage: | ||
| 71 | p = HTMLParser() | ||
| 72 | p.feed(data) | ||
| 73 | ... | ||
| 74 | p.close() | ||
| 75 | |||
| 76 | Start tags are handled by calling self.handle_starttag() or | ||
| 77 | self.handle_startendtag(); end tags by self.handle_endtag(). The | ||
| 78 | data between tags is passed from the parser to the derived class | ||
| 79 | by calling self.handle_data() with the data as argument (the data | ||
| 80 | may be split up in arbitrary chunks). Entity references are | ||
| 81 | passed by calling self.handle_entityref() with the entity | ||
| 82 | reference as the argument. Numeric character references are | ||
| 83 | passed to self.handle_charref() with the string containing the | ||
| 84 | reference as the argument. | ||
| 85 | """ | ||
| 86 | |||
| 87 | CDATA_CONTENT_ELEMENTS = ("script", "style") | ||
| 88 | |||
| 89 | |||
| 90 | def __init__(self): | ||
| 91 | """Initialize and reset this instance.""" | ||
| 92 | self.reset() | ||
| 93 | |||
| 94 | def reset(self): | ||
| 95 | """Reset this instance. Loses all unprocessed data.""" | ||
| 96 | self.rawdata = '' | ||
| 97 | self.lasttag = '???' | ||
| 98 | self.interesting = interesting_normal | ||
| 99 | markupbase.ParserBase.reset(self) | ||
| 100 | |||
| 101 | def feed(self, data): | ||
| 102 | """Feed data to the parser. | ||
| 103 | |||
| 104 | Call this as often as you want, with as little or as much text | ||
| 105 | as you want (may include '\n'). | ||
| 106 | """ | ||
| 107 | self.rawdata = self.rawdata + data | ||
| 108 | self.goahead(0) | ||
| 109 | |||
| 110 | def close(self): | ||
| 111 | """Handle any buffered data.""" | ||
| 112 | self.goahead(1) | ||
| 113 | |||
| 114 | def error(self, message): | ||
| 115 | raise HTMLParseError(message, self.getpos()) | ||
| 116 | |||
| 117 | __starttag_text = None | ||
| 118 | |||
| 119 | def get_starttag_text(self): | ||
| 120 | """Return full source of start tag: '<...>'.""" | ||
| 121 | return self.__starttag_text | ||
| 122 | |||
| 123 | def set_cdata_mode(self): | ||
| 124 | self.interesting = interesting_cdata | ||
| 125 | |||
| 126 | def clear_cdata_mode(self): | ||
| 127 | self.interesting = interesting_normal | ||
| 128 | |||
| 129 | # Internal -- handle data as far as reasonable. May leave state | ||
| 130 | # and data to be processed by a subsequent call. If 'end' is | ||
| 131 | # true, force handling all data as if followed by EOF marker. | ||
| 132 | def goahead(self, end): | ||
| 133 | rawdata = self.rawdata | ||
| 134 | i = 0 | ||
| 135 | n = len(rawdata) | ||
| 136 | while i < n: | ||
| 137 | match = self.interesting.search(rawdata, i) # < or & | ||
| 138 | if match: | ||
| 139 | j = match.start() | ||
| 140 | else: | ||
| 141 | j = n | ||
| 142 | if i < j: self.handle_data(rawdata[i:j]) | ||
| 143 | i = self.updatepos(i, j) | ||
| 144 | if i == n: break | ||
| 145 | startswith = rawdata.startswith | ||
| 146 | if startswith('<', i): | ||
| 147 | if starttagopen.match(rawdata, i): # < + letter | ||
| 148 | k = self.parse_starttag(i) | ||
| 149 | elif startswith("</", i): | ||
| 150 | k = self.parse_endtag(i) | ||
| 151 | elif startswith("<!--", i): | ||
| 152 | k = self.parse_comment(i) | ||
| 153 | elif startswith("<?", i): | ||
| 154 | k = self.parse_pi(i) | ||
| 155 | elif startswith("<!", i): | ||
| 156 | k = self.parse_declaration(i) | ||
| 157 | elif (i + 1) < n: | ||
| 158 | self.handle_data("<") | ||
| 159 | k = i + 1 | ||
| 160 | else: | ||
| 161 | break | ||
| 162 | if k < 0: | ||
| 163 | if end: | ||
| 164 | self.error("EOF in middle of construct") | ||
| 165 | break | ||
| 166 | i = self.updatepos(i, k) | ||
| 167 | elif startswith("&#", i): | ||
| 168 | match = charref.match(rawdata, i) | ||
| 169 | if match: | ||
| 170 | name = match.group()[2:-1] | ||
| 171 | self.handle_charref(name) | ||
| 172 | k = match.end() | ||
| 173 | if not startswith(';', k-1): | ||
| 174 | k = k - 1 | ||
| 175 | i = self.updatepos(i, k) | ||
| 176 | continue | ||
| 177 | else: | ||
| 178 | break | ||
| 179 | elif startswith('&', i): | ||
| 180 | match = entityref.match(rawdata, i) | ||
| 181 | if match: | ||
| 182 | name = match.group(1) | ||
| 183 | self.handle_entityref(name) | ||
| 184 | k = match.end() | ||
| 185 | if not startswith(';', k-1): | ||
| 186 | k = k - 1 | ||
| 187 | i = self.updatepos(i, k) | ||
| 188 | continue | ||
| 189 | match = incomplete.match(rawdata, i) | ||
| 190 | if match: | ||
| 191 | # match.group() will contain at least 2 chars | ||
| 192 | if end and match.group() == rawdata[i:]: | ||
| 193 | self.error("EOF in middle of entity or char ref") | ||
| 194 | # incomplete | ||
| 195 | break | ||
| 196 | elif (i + 1) < n: | ||
| 197 | # not the end of the buffer, and can't be confused | ||
| 198 | # with some other construct | ||
| 199 | self.handle_data("&") | ||
| 200 | i = self.updatepos(i, i + 1) | ||
| 201 | else: | ||
| 202 | break | ||
| 203 | else: | ||
| 204 | assert 0, "interesting.search() lied" | ||
| 205 | # end while | ||
| 206 | if end and i < n: | ||
| 207 | self.handle_data(rawdata[i:n]) | ||
| 208 | i = self.updatepos(i, n) | ||
| 209 | self.rawdata = rawdata[i:] | ||
| 210 | |||
| 211 | # Internal -- parse processing instr, return end or -1 if not terminated | ||
| 212 | def parse_pi(self, i): | ||
| 213 | rawdata = self.rawdata | ||
| 214 | assert rawdata[i:i+2] == '<?', 'unexpected call to parse_pi()' | ||
| 215 | match = piclose.search(rawdata, i+2) # > | ||
| 216 | if not match: | ||
| 217 | return -1 | ||
| 218 | j = match.start() | ||
| 219 | self.handle_pi(rawdata[i+2: j]) | ||
| 220 | j = match.end() | ||
| 221 | return j | ||
| 222 | |||
| 223 | # Internal -- handle starttag, return end or -1 if not terminated | ||
| 224 | def parse_starttag(self, i): | ||
| 225 | self.__starttag_text = None | ||
| 226 | endpos = self.check_for_whole_start_tag(i) | ||
| 227 | if endpos < 0: | ||
| 228 | return endpos | ||
| 229 | rawdata = self.rawdata | ||
| 230 | self.__starttag_text = rawdata[i:endpos] | ||
| 231 | |||
| 232 | # Now parse the data between i+1 and j into a tag and attrs | ||
| 233 | attrs = [] | ||
| 234 | match = tagfind.match(rawdata, i+1) | ||
| 235 | assert match, 'unexpected call to parse_starttag()' | ||
| 236 | k = match.end() | ||
| 237 | self.lasttag = tag = rawdata[i+1:k].lower() | ||
| 238 | |||
| 239 | while k < endpos: | ||
| 240 | m = attrfind.match(rawdata, k) | ||
| 241 | if not m: | ||
| 242 | break | ||
| 243 | attrname, rest, attrvalue = m.group(1, 2, 3) | ||
| 244 | if not rest: | ||
| 245 | attrvalue = None | ||
| 246 | elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ | ||
| 247 | attrvalue[:1] == '"' == attrvalue[-1:]: | ||
| 248 | attrvalue = attrvalue[1:-1] | ||
| 249 | attrvalue = self.unescape(attrvalue) | ||
| 250 | attrs.append((attrname.lower(), attrvalue)) | ||
| 251 | k = m.end() | ||
| 252 | |||
| 253 | end = rawdata[k:endpos].strip() | ||
| 254 | if end not in (">", "/>"): | ||
| 255 | lineno, offset = self.getpos() | ||
| 256 | if "\n" in self.__starttag_text: | ||
| 257 | lineno = lineno + self.__starttag_text.count("\n") | ||
| 258 | offset = len(self.__starttag_text) \ | ||
| 259 | - self.__starttag_text.rfind("\n") | ||
| 260 | else: | ||
| 261 | offset = offset + len(self.__starttag_text) | ||
| 262 | self.error("junk characters in start tag: %r" | ||
| 263 | % (rawdata[k:endpos][:20],)) | ||
| 264 | if end.endswith('/>'): | ||
| 265 | # XHTML-style empty tag: <span attr="value" /> | ||
| 266 | self.handle_startendtag(tag, attrs) | ||
| 267 | else: | ||
| 268 | self.handle_starttag(tag, attrs) | ||
| 269 | if tag in self.CDATA_CONTENT_ELEMENTS: | ||
| 270 | self.set_cdata_mode() | ||
| 271 | return endpos | ||
| 272 | |||
| 273 | # Internal -- check to see if we have a complete starttag; return end | ||
| 274 | # or -1 if incomplete. | ||
| 275 | def check_for_whole_start_tag(self, i): | ||
| 276 | rawdata = self.rawdata | ||
| 277 | m = locatestarttagend.match(rawdata, i) | ||
| 278 | if m: | ||
| 279 | j = m.end() | ||
| 280 | next = rawdata[j:j+1] | ||
| 281 | if next == ">": | ||
| 282 | return j + 1 | ||
| 283 | if next == "/": | ||
| 284 | if rawdata.startswith("/>", j): | ||
| 285 | return j + 2 | ||
| 286 | if rawdata.startswith("/", j): | ||
| 287 | # buffer boundary | ||
| 288 | return -1 | ||
| 289 | # else bogus input | ||
| 290 | self.updatepos(i, j + 1) | ||
| 291 | self.error("malformed empty start tag") | ||
| 292 | if next == "": | ||
| 293 | # end of input | ||
| 294 | return -1 | ||
| 295 | if next in ("abcdefghijklmnopqrstuvwxyz=/" | ||
| 296 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ"): | ||
| 297 | # end of input in or before attribute value, or we have the | ||
| 298 | # '/' from a '/>' ending | ||
| 299 | return -1 | ||
| 300 | self.updatepos(i, j) | ||
| 301 | self.error("malformed start tag") | ||
| 302 | raise AssertionError("we should not get here!") | ||
| 303 | |||
| 304 | # Internal -- parse endtag, return end or -1 if incomplete | ||
| 305 | def parse_endtag(self, i): | ||
| 306 | rawdata = self.rawdata | ||
| 307 | assert rawdata[i:i+2] == "</", "unexpected call to parse_endtag" | ||
| 308 | match = endendtag.search(rawdata, i+1) # > | ||
| 309 | if not match: | ||
| 310 | return -1 | ||
| 311 | j = match.end() | ||
| 312 | match = endtagfind.match(rawdata, i) # </ + tag + > | ||
| 313 | if not match: | ||
| 314 | self.error("bad end tag: %r" % (rawdata[i:j],)) | ||
| 315 | tag = match.group(1) | ||
| 316 | self.handle_endtag(tag.lower()) | ||
| 317 | self.clear_cdata_mode() | ||
| 318 | return j | ||
| 319 | |||
| 320 | # Overridable -- finish processing of start+end tag: <tag.../> | ||
| 321 | def handle_startendtag(self, tag, attrs): | ||
| 322 | self.handle_starttag(tag, attrs) | ||
| 323 | self.handle_endtag(tag) | ||
| 324 | |||
| 325 | # Overridable -- handle start tag | ||
| 326 | def handle_starttag(self, tag, attrs): | ||
| 327 | pass | ||
| 328 | |||
| 329 | # Overridable -- handle end tag | ||
| 330 | def handle_endtag(self, tag): | ||
| 331 | pass | ||
| 332 | |||
| 333 | # Overridable -- handle character reference | ||
| 334 | def handle_charref(self, name): | ||
| 335 | pass | ||
| 336 | |||
| 337 | # Overridable -- handle entity reference | ||
| 338 | def handle_entityref(self, name): | ||
| 339 | pass | ||
| 340 | |||
| 341 | # Overridable -- handle data | ||
| 342 | def handle_data(self, data): | ||
| 343 | pass | ||
| 344 | |||
| 345 | # Overridable -- handle comment | ||
| 346 | def handle_comment(self, data): | ||
| 347 | pass | ||
| 348 | |||
| 349 | # Overridable -- handle declaration | ||
| 350 | def handle_decl(self, decl): | ||
| 351 | pass | ||
| 352 | |||
| 353 | # Overridable -- handle processing instruction | ||
| 354 | def handle_pi(self, data): | ||
| 355 | pass | ||
| 356 | |||
| 357 | def unknown_decl(self, data): | ||
| 358 | self.error("unknown declaration: %r" % (data,)) | ||
| 359 | |||
| 360 | # Internal -- helper to remove special character quoting | ||
| 361 | def unescape(self, s): | ||
| 362 | if '&' not in s: | ||
| 363 | return s | ||
| 364 | s = s.replace("<", "<") | ||
| 365 | s = s.replace(">", ">") | ||
| 366 | s = s.replace("'", "'") | ||
| 367 | s = s.replace(""", '"') | ||
| 368 | s = s.replace("&", "&") # Must be last | ||
| 369 | return s | ||