#!/usr/bin/env python3 # This script transforms markdown files into html and (optionally) nroff. The# output files are written into the current directory named for the input file# without the .md suffix and either the .html suffix or no suffix.## If the input .md file has a section number at the end of the name (e.g.,# rsync.1.md) a nroff file is also output (PROJ.NUM.md -> PROJ.NUM).## The markdown input format has one extra extension: if a numbered list starts# at 0, it is turned into a description list. The dl's dt tag is taken from the# contents of the first tag inside the li, which is usually a p, code, or# strong tag.## The cmarkgfm or commonmark lib is used to transforms the input file into# html. Then, the html.parser is used as a state machine that lets us tweak# the html and (optionally) output nroff data based on the html tags.## If the string @USE_GFM_PARSER@ exists in the file, the string is removed and# a github-flavored-markup parser is used to parse the file.## The man-page .md files also get the vars @VERSION@, @BINDIR@, and @LIBDIR@# substituted. Some of these values depend on the Makefile $(prefix) (see the# generated Makefile). If the maintainer wants to build files for /usr/local# while creating release-ready man-page files for /usr, use the environment to# set RSYNC_OVERRIDE_PREFIX=/usr. # Copyright (C) 2020 - 2021 Wayne Davison## This program is freely redistributable. import os, sys, re, argparse, subprocess, timefrom html.parser import HTMLParser VALID_PAGES = 'README INSTALL COPYING rsync.1 rrsync.1 rsync-ssl.1 rsyncd.conf.5'.split() CONSUMES_TXT = set('h1 h2 h3 p li pre'.split()) HTML_START = """\<html><head><title>%TITLE%</title><meta charset="UTF-8"/><link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet"><style>body { max-width: 50em; margin: auto;}body, b, strong, u { font-family: 'Roboto', sans-serif;}a.tgt { font-face: symbol; font-weight: 400; font-size: 70%; visibility: hidden; text-decoration: none; color: #ddd; padding: 0 4px; border: 0; }a.tgt:after { content: '🔗'; }a.tgt:hover { color: #444; background-color: #eaeaea; }h1:hover > a.tgt, h2:hover > a.tgt, h3:hover > a.tgt, dt:hover > a.tgt { visibility: visible; }code { font-family: 'Roboto Mono', monospace; font-weight: bold; white-space: pre;}pre code { display: block; font-weight: normal;}blockquote pre code { background: #f1f1f1;}dd p:first-of-type { margin-block-start: 0em;}</style></head><body>""" TABLE_STYLE = """\table { border-color: grey; border-spacing: 0;}tr { border-top: 1px solid grey;}tr:nth-child(2n) { background-color: #f6f8fa;}th, td { border: 1px solid #dfe2e5; text-align: center; padding-left: 1em; padding-right: 1em;}""" MAN_HTML_END = """\<div style="float: right"><p><i>%s</i></p></div>""" HTML_END = """\</body></html>""" MAN_START = r""".TH "%s" "%s" "%s" "%s" "User Commands".\" prefix=%s""".lstrip() MAN_END = """\""" NORM_FONT = ('\1', r"\fP")BOLD_FONT = ('\2', r"\fB")UNDR_FONT = ('\3', r"\fI")NBR_DASH = ('\4', r"\-")NBR_SPACE = ('\xa0', r"\ ") FILENAME_RE = re.compile(r'^(?P<fn>(?P<srcdir>.+/)?(?P<name>(?P<prog>[^/]+?)(\.(?P<sect>\d+))?)\.md)$')ASSIGNMENT_RE = re.compile(r'^(\w+)=(.+)')VER_RE = re.compile(r'^#define\s+RSYNC_VERSION\s+"(\d.+?)"', re.M)TZ_RE = re.compile(r'^#define\s+MAINTAINER_TZ_OFFSET\s+(-?\d+(\.\d+)?)', re.M)VAR_REF_RE = re.compile(r'\$\{(\w+)\}')VERSION_RE = re.compile(r' (\d[.\d]+)[, ]')BIN_CHARS_RE = re.compile(r'[\1-\7]+')LONG_OPT_DASH_RE = re.compile(r'(--\w[-\w]+)')SPACE_DOUBLE_DASH_RE = re.compile(r'\s--(\s)')NON_SPACE_SINGLE_DASH_RE = re.compile(r'(^|\W)-')WHITESPACE_RE = re.compile(r'\s')CODE_BLOCK_RE = re.compile(r'[%s]([^=%s]+)[=%s]' % (BOLD_FONT[0], NORM_FONT[0], NORM_FONT[0]))NBR_DASH_RE = re.compile(r'[%s]' % NBR_DASH[0])INVALID_TARGET_CHARS_RE = re.compile(r'[^-A-Za-z0-9._]')INVALID_START_CHAR_RE = re.compile(r'^([^A-Za-z0-9])')MANIFY_LINESTART_RE = re.compile(r"^(['.])", flags=re.M) md_parser = Noneenv_subs = { } warning_count = 0 def main(): for mdfn in args.mdfiles: parse_md_file(mdfn) if args.test: print("The test was successful.") def parse_md_file(mdfn): fi = FILENAME_RE.match(mdfn) if not fi: die('Failed to parse a md input file name:', mdfn) fi = argparse.Namespace(**fi.groupdict()) fi.want_manpage = not not fi.sect if fi.want_manpage: fi.title = fi.prog + '(' + fi.sect + ') manpage' else: fi.title = fi.prog + ' for rsync' if fi.want_manpage: if not env_subs: find_man_substitutions() prog_ver = 'rsync ' + env_subs['VERSION'] if fi.prog != 'rsync': prog_ver = fi.prog + ' from ' + prog_ver fi.man_headings = (fi.prog, fi.sect, env_subs['date'], prog_ver, env_subs['prefix']) with open(mdfn, 'r', encoding='utf-8') as fh: txt = fh.read() use_gfm_parser = '@USE_GFM_PARSER@' in txt if use_gfm_parser: txt = txt.replace('@USE_GFM_PARSER@', '') if fi.want_manpage: txt = (txt.replace('@VERSION@', env_subs['VERSION']) .replace('@BINDIR@', env_subs['bindir']) .replace('@LIBDIR@', env_subs['libdir'])) if use_gfm_parser: if not gfm_parser: die('Input file requires cmarkgfm parser:', mdfn) fi.html_in = gfm_parser(txt) else: fi.html_in = md_parser(txt) txt = None TransformHtml(fi) if args.test: return output_list = [ (fi.name + '.html', fi.html_out) ] if fi.want_manpage: output_list += [ (fi.name, fi.man_out) ] for fn, txt in output_list: if args.dest and args.dest != '.': fn = os.path.join(args.dest, fn) if os.path.lexists(fn): os.unlink(fn) print("Wrote:", fn) with open(fn, 'w', encoding='utf-8') as fh: fh.write(txt) def find_man_substitutions(): srcdir = os.path.dirname(sys.argv[0]) + '/' mtime = 0 git_dir = srcdir + '.git' if os.path.lexists(git_dir): mtime = int(subprocess.check_output(['git', '--git-dir', git_dir, 'log', '-1', '--format=%at'])) # Allow "prefix" to be overridden via the environment: env_subs['prefix'] = os.environ.get('RSYNC_OVERRIDE_PREFIX', None) if args.test: env_subs['VERSION'] = '1.0.0' env_subs['bindir'] = '/usr/bin' env_subs['libdir'] = '/usr/lib/rsync' tz_offset = 0 else: for fn in (srcdir + 'version.h', 'Makefile'): try: st = os.lstat(fn) except OSError: die('Failed to find', srcdir + fn) if not mtime: mtime = st.st_mtime with open(srcdir + 'version.h', 'r', encoding='utf-8') as fh: txt = fh.read() m = VER_RE.search(txt) env_subs['VERSION'] = m.group(1) m = TZ_RE.search(txt) # the tzdata lib may not be installed, so we use a simple hour offset tz_offset = float(m.group(1)) * 60 * 60 with open('Makefile', 'r', encoding='utf-8') as fh: for line in fh: m = ASSIGNMENT_RE.match(line) if not m: continue var, val = (m.group(1), m.group(2)) if var == 'prefix' and env_subs[var] is not None: continue while VAR_REF_RE.search(val): val = VAR_REF_RE.sub(lambda m: env_subs[m.group(1)], val) env_subs[var] = val if var == 'srcdir': break env_subs['date'] = time.strftime('%d %b %Y', time.gmtime(mtime + tz_offset)).lstrip('0') if 'SOURCE_DATE_EPOCH' in os.environ: env_subs['date'] = time.strftime('%d %b %Y', time.gmtime(int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))) def html_via_commonmark(txt): return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt)) class TransformHtml(HTMLParser): def __init__(self, fi): HTMLParser.__init__(self, convert_charrefs=True) self.fn = fi.fn st = self.state = argparse.Namespace( list_state = [ ], p_macro = ".P\n", at_first_tag_in_li = False, at_first_tag_in_dd = False, dt_from = None, in_pre = False, in_code = False, html_out = [ HTML_START.replace('%TITLE%', fi.title) ], man_out = [ ], txt = '', want_manpage = fi.want_manpage, created_hashtags = set(), derived_hashtags = set(), referenced_hashtags = set(), bad_hashtags = set(), latest_targets = [ ], opt_prefix = 'opt', a_href = None, a_href_external = False, a_txt_start = None, after_a_tag = False, target_suf = '', ) if st.want_manpage: st.man_out.append(MAN_START % fi.man_headings) if '</table>' in fi.html_in: st.html_out[0] = st.html_out[0].replace('</style>', TABLE_STYLE + '</style>') self.feed(fi.html_in) fi.html_in = None if st.want_manpage: st.html_out.append(MAN_HTML_END % env_subs['date']) st.html_out.append(HTML_END) st.man_out.append(MAN_END) fi.html_out = ''.join(st.html_out) st.html_out = None fi.man_out = ''.join(st.man_out) st.man_out = None for tgt, txt in st.derived_hashtags: derived = txt2target(txt, tgt) if derived not in st.created_hashtags: txt = BIN_CHARS_RE.sub('', txt.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ')) warn('Unknown derived hashtag link in', self.fn, 'based on:', (tgt, txt)) for bad in st.bad_hashtags: if bad in st.created_hashtags: warn('Missing "#" in hashtag link in', self.fn + ':', bad) else: warn('Unknown non-hashtag link in', self.fn + ':', bad) for bad in st.referenced_hashtags - st.created_hashtags: warn('Unknown hashtag link in', self.fn + ':', '#' + bad) def handle_UE(self): st = self.state if st.txt.startswith(('.', ',', '!', '?', ';', ':')): st.man_out[-1] = ".UE " + st.txt[0] + "\n" st.txt = st.txt[1:] st.after_a_tag = False def handle_starttag(self, tag, attrs_list): st = self.state if args.debug: self.output_debug('START', (tag, attrs_list)) if st.at_first_tag_in_li: if st.list_state[-1] == 'dl': st.dt_from = tag if tag == 'p': tag = 'dt' else: st.html_out.append('<dt>') elif tag == 'p': st.at_first_tag_in_dd = True # Kluge to suppress a .P at the start of an li. st.at_first_tag_in_li = False if tag == 'p': if not st.at_first_tag_in_dd: st.man_out.append(st.p_macro) elif tag == 'li': st.at_first_tag_in_li = True lstate = st.list_state[-1] if lstate == 'dl': return if lstate == 'o': st.man_out.append(".IP o\n") else: st.man_out.append(".IP " + str(lstate) + ".\n") st.list_state[-1] += 1 elif tag == 'blockquote': st.man_out.append(".RS 4\n") elif tag == 'pre': st.in_pre = True st.man_out.append(st.p_macro + ".nf\n") elif tag == 'code' and not st.in_pre: st.in_code = True st.txt += BOLD_FONT[0] elif tag == 'strong' or tag == 'b': st.txt += BOLD_FONT[0] elif tag == 'em' or tag == 'i': if st.want_manpage: tag = 'u' # Change it into underline to be more like the manpage st.txt += UNDR_FONT[0] elif tag == 'ol': start = 1 for var, val in attrs_list: if var == 'start': start = int(val) # We only support integers. break if st.list_state: st.man_out.append(".RS\n") if start == 0: tag = 'dl' attrs_list = [ ] st.list_state.append('dl') else: st.list_state.append(start) st.man_out.append(st.p_macro) st.p_macro = ".IP\n" elif tag == 'ul': st.man_out.append(st.p_macro) if st.list_state: st.man_out.append(".RS\n") st.p_macro = ".IP\n" st.list_state.append('o') elif tag == 'hr': st.man_out.append(".l\n") st.html_out.append("<hr />") return elif tag == 'a': st.a_href = None for var, val in attrs_list: if var == 'href': if val.startswith(('https://', 'http://', 'mailto:', 'ftp:')): if st.after_a_tag: self.handle_UE() st.man_out.append(manify(st.txt.strip()) + "\n") st.man_out.append(".UR " + val + "\n") st.txt = '' st.a_href = val st.a_href_external = True elif '#' in val: pg, tgt = val.split('#', 1) if pg and pg not in VALID_PAGES or '#' in tgt: st.bad_hashtags.add(val) elif tgt in ('', 'opt', 'dopt'): st.a_href = val st.a_href_external = False elif pg == '': st.referenced_hashtags.add(tgt) if tgt in st.latest_targets: warn('Found link to the current section in', self.fn + ':', val) elif val not in VALID_PAGES: st.bad_hashtags.add(val) st.a_txt_start = len(st.txt) st.html_out.append('<' + tag + ''.join(' ' + var + '="' + htmlify(val) + '"' for var, val in attrs_list) + '>') st.at_first_tag_in_dd = False def handle_endtag(self, tag): st = self.state if args.debug: self.output_debug('END', (tag,)) if st.after_a_tag: self.handle_UE() if tag in CONSUMES_TXT or st.dt_from == tag: txt = st.txt.strip() st.txt = '' else: txt = None add_to_txt = None if tag == 'h1': tgt = txt target_suf = '' if tgt.startswith('NEWS for '): m = VERSION_RE.search(tgt) if m: tgt = m.group(1) st.target_suf = '-' + tgt self.add_targets(tag, tgt) elif tag == 'h2': st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n') self.add_targets(tag, txt, st.target_suf) st.opt_prefix = 'dopt' if txt == 'DAEMON OPTIONS' else 'opt' elif tag == 'h3': st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n') self.add_targets(tag, txt, st.target_suf) elif tag == 'p': if st.dt_from == 'p': tag = 'dt' st.man_out.append('.IP "' + manify(txt) + '"\n') if txt.startswith(BOLD_FONT[0]): self.add_targets(tag, txt) st.dt_from = None elif txt != '': st.man_out.append(manify(txt) + "\n") elif tag == 'li': if st.list_state[-1] == 'dl': if st.at_first_tag_in_li: die("Invalid 0. -> td translation") tag = 'dd' if txt != '': st.man_out.append(manify(txt) + "\n") st.at_first_tag_in_li = False elif tag == 'blockquote': st.man_out.append(".RE\n") elif tag == 'pre': st.in_pre = False st.man_out.append(manify(txt) + "\n.fi\n") elif (tag == 'code' and not st.in_pre): st.in_code = False add_to_txt = NORM_FONT[0] elif tag == 'strong' or tag == 'b': add_to_txt = NORM_FONT[0] elif tag == 'em' or tag == 'i': if st.want_manpage: tag = 'u' # Change it into underline to be more like the manpage add_to_txt = NORM_FONT[0] elif tag == 'ol' or tag == 'ul': if st.list_state.pop() == 'dl': tag = 'dl' if st.list_state: st.man_out.append(".RE\n") else: st.p_macro = ".P\n" st.at_first_tag_in_dd = False elif tag == 'hr': return elif tag == 'a': if st.a_href_external: st.txt = st.txt.strip()Lines 1–500 of 667. Load more
#!/usr/bin/env python3 # This script transforms markdown files into html and (optionally) nroff. The # output files are written into the current directory named for the input file # without the .md suffix and either the .html suffix or no suffix. # # If the input .md file has a section number at the end of the name (e.g., # rsync.1.md) a nroff file is also output (PROJ.NUM.md -> PROJ.NUM). # # The markdown input format has one extra extension: if a numbered list starts # at 0, it is turned into a description list. The dl's dt tag is taken from the # contents of the first tag inside the li, which is usually a p, code, or # strong tag. # # The cmarkgfm or commonmark lib is used to transforms the input file into # html. Then, the html.parser is used as a state machine that lets us tweak # the html and (optionally) output nroff data based on the html tags. # # If the string @USE_GFM_PARSER@ exists in the file, the string is removed and # a github-flavored-markup parser is used to parse the file. # # The man-page .md files also get the vars @VERSION@, @BINDIR@, and @LIBDIR@ # substituted. Some of these values depend on the Makefile $(prefix) (see the # generated Makefile). If the maintainer wants to build files for /usr/local # while creating release-ready man-page files for /usr, use the environment to # set RSYNC_OVERRIDE_PREFIX=/usr. # Copyright (C) 2020 - 2021 Wayne Davison # # This program is freely redistributable. import os, sys, re, argparse, subprocess, time from html.parser import HTMLParser VALID_PAGES = 'README INSTALL COPYING rsync.1 rrsync.1 rsync-ssl.1 rsyncd.conf.5'.split() CONSUMES_TXT = set('h1 h2 h3 p li pre'.split()) HTML_START = """\ <html><head> <title>%TITLE%</title> <meta charset="UTF-8"/> <link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet"> <style> body { max-width: 50em; margin: auto; } body, b, strong, u { font-family: 'Roboto', sans-serif; } a.tgt { font-face: symbol; font-weight: 400; font-size: 70%; visibility: hidden; text-decoration: none; color: #ddd; padding: 0 4px; border: 0; } a.tgt:after { content: '🔗'; } a.tgt:hover { color: #444; background-color: #eaeaea; } h1:hover > a.tgt, h2:hover > a.tgt, h3:hover > a.tgt, dt:hover > a.tgt { visibility: visible; } code { font-family: 'Roboto Mono', monospace; font-weight: bold; white-space: pre; } pre code { display: block; font-weight: normal; } blockquote pre code { background: #f1f1f1; } dd p:first-of-type { margin-block-start: 0em; } </style> </head><body> """ TABLE_STYLE = """\ table { border-color: grey; border-spacing: 0; } tr { border-top: 1px solid grey; } tr:nth-child(2n) { background-color: #f6f8fa; } th, td { border: 1px solid #dfe2e5; text-align: center; padding-left: 1em; padding-right: 1em; } """ MAN_HTML_END = """\ <div style="float: right"><p><i>%s</i></p></div> """ HTML_END = """\ </body></html> """ MAN_START = r""" .TH "%s" "%s" "%s" "%s" "User Commands" .\" prefix=%s """.lstrip() MAN_END = """\ """ NORM_FONT = ('\1', r"\fP") BOLD_FONT = ('\2', r"\fB") UNDR_FONT = ('\3', r"\fI") NBR_DASH = ('\4', r"\-") NBR_SPACE = ('\xa0', r"\ ") FILENAME_RE = re.compile(r'^(?P<fn>(?P<srcdir>.+/)?(?P<name>(?P<prog>[^/]+?)(\.(?P<sect>\d+))?)\.md)$') ASSIGNMENT_RE = re.compile(r'^(\w+)=(.+)') VER_RE = re.compile(r'^#define\s+RSYNC_VERSION\s+"(\d.+?)"', re.M) TZ_RE = re.compile(r'^#define\s+MAINTAINER_TZ_OFFSET\s+(-?\d+(\.\d+)?)', re.M) VAR_REF_RE = re.compile(r'\$\{(\w+)\}') VERSION_RE = re.compile(r' (\d[.\d]+)[, ]') BIN_CHARS_RE = re.compile(r'[\1-\7]+') LONG_OPT_DASH_RE = re.compile(r'(--\w[-\w]+)') SPACE_DOUBLE_DASH_RE = re.compile(r'\s--(\s)') NON_SPACE_SINGLE_DASH_RE = re.compile(r'(^|\W)-') WHITESPACE_RE = re.compile(r'\s') CODE_BLOCK_RE = re.compile(r'[%s]([^=%s]+)[=%s]' % (BOLD_FONT[0], NORM_FONT[0], NORM_FONT[0])) NBR_DASH_RE = re.compile(r'[%s]' % NBR_DASH[0]) INVALID_TARGET_CHARS_RE = re.compile(r'[^-A-Za-z0-9._]') INVALID_START_CHAR_RE = re.compile(r'^([^A-Za-z0-9])') MANIFY_LINESTART_RE = re.compile(r"^(['.])", flags=re.M) md_parser = None env_subs = { } warning_count = 0 def main(): for mdfn in args.mdfiles: parse_md_file(mdfn) if args.test: print("The test was successful.") def parse_md_file(mdfn): fi = FILENAME_RE.match(mdfn) if not fi: die('Failed to parse a md input file name:', mdfn) fi = argparse.Namespace(**fi.groupdict()) fi.want_manpage = not not fi.sect if fi.want_manpage: fi.title = fi.prog + '(' + fi.sect + ') manpage' else: fi.title = fi.prog + ' for rsync' if fi.want_manpage: if not env_subs: find_man_substitutions() prog_ver = 'rsync ' + env_subs['VERSION'] if fi.prog != 'rsync': prog_ver = fi.prog + ' from ' + prog_ver fi.man_headings = (fi.prog, fi.sect, env_subs['date'], prog_ver, env_subs['prefix']) with open(mdfn, 'r', encoding='utf-8') as fh: txt = fh.read() use_gfm_parser = '@USE_GFM_PARSER@' in txt if use_gfm_parser: txt = txt.replace('@USE_GFM_PARSER@', '') if fi.want_manpage: txt = (txt.replace('@VERSION@', env_subs['VERSION']) .replace('@BINDIR@', env_subs['bindir']) .replace('@LIBDIR@', env_subs['libdir'])) if use_gfm_parser: if not gfm_parser: die('Input file requires cmarkgfm parser:', mdfn) fi.html_in = gfm_parser(txt) else: fi.html_in = md_parser(txt) txt = None TransformHtml(fi) if args.test: return output_list = [ (fi.name + '.html', fi.html_out) ] if fi.want_manpage: output_list += [ (fi.name, fi.man_out) ] for fn, txt in output_list: if args.dest and args.dest != '.': fn = os.path.join(args.dest, fn) if os.path.lexists(fn): os.unlink(fn) print("Wrote:", fn) with open(fn, 'w', encoding='utf-8') as fh: fh.write(txt) def find_man_substitutions(): srcdir = os.path.dirname(sys.argv[0]) + '/' mtime = 0 git_dir = srcdir + '.git' if os.path.lexists(git_dir): mtime = int(subprocess.check_output(['git', '--git-dir', git_dir, 'log', '-1', '--format=%at'])) # Allow "prefix" to be overridden via the environment: env_subs['prefix'] = os.environ.get('RSYNC_OVERRIDE_PREFIX', None) if args.test: env_subs['VERSION'] = '1.0.0' env_subs['bindir'] = '/usr/bin' env_subs['libdir'] = '/usr/lib/rsync' tz_offset = 0 else: for fn in (srcdir + 'version.h', 'Makefile'): try: st = os.lstat(fn) except OSError: die('Failed to find', srcdir + fn) if not mtime: mtime = st.st_mtime with open(srcdir + 'version.h', 'r', encoding='utf-8') as fh: txt = fh.read() m = VER_RE.search(txt) env_subs['VERSION'] = m.group(1) m = TZ_RE.search(txt) # the tzdata lib may not be installed, so we use a simple hour offset tz_offset = float(m.group(1)) * 60 * 60 with open('Makefile', 'r', encoding='utf-8') as fh: for line in fh: m = ASSIGNMENT_RE.match(line) if not m: continue var, val = (m.group(1), m.group(2)) if var == 'prefix' and env_subs[var] is not None: continue while VAR_REF_RE.search(val): val = VAR_REF_RE.sub(lambda m: env_subs[m.group(1)], val) env_subs[var] = val if var == 'srcdir': break env_subs['date'] = time.strftime('%d %b %Y', time.gmtime(mtime + tz_offset)).lstrip('0') if 'SOURCE_DATE_EPOCH' in os.environ: env_subs['date'] = time.strftime('%d %b %Y', time.gmtime(int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))) def html_via_commonmark(txt): return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt)) class TransformHtml(HTMLParser): def __init__(self, fi): HTMLParser.__init__(self, convert_charrefs=True) self.fn = fi.fn st = self.state = argparse.Namespace( list_state = [ ], p_macro = ".P\n", at_first_tag_in_li = False, at_first_tag_in_dd = False, dt_from = None, in_pre = False, in_code = False, html_out = [ HTML_START.replace('%TITLE%', fi.title) ], man_out = [ ], txt = '', want_manpage = fi.want_manpage, created_hashtags = set(), derived_hashtags = set(), referenced_hashtags = set(), bad_hashtags = set(), latest_targets = [ ], opt_prefix = 'opt', a_href = None, a_href_external = False, a_txt_start = None, after_a_tag = False, target_suf = '', ) if st.want_manpage: st.man_out.append(MAN_START % fi.man_headings) if '</table>' in fi.html_in: st.html_out[0] = st.html_out[0].replace('</style>', TABLE_STYLE + '</style>') self.feed(fi.html_in) fi.html_in = None if st.want_manpage: st.html_out.append(MAN_HTML_END % env_subs['date']) st.html_out.append(HTML_END) st.man_out.append(MAN_END) fi.html_out = ''.join(st.html_out) st.html_out = None fi.man_out = ''.join(st.man_out) st.man_out = None for tgt, txt in st.derived_hashtags: derived = txt2target(txt, tgt) if derived not in st.created_hashtags: txt = BIN_CHARS_RE.sub('', txt.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ')) warn('Unknown derived hashtag link in', self.fn, 'based on:', (tgt, txt)) for bad in st.bad_hashtags: if bad in st.created_hashtags: warn('Missing "#" in hashtag link in', self.fn + ':', bad) else: warn('Unknown non-hashtag link in', self.fn + ':', bad) for bad in st.referenced_hashtags - st.created_hashtags: warn('Unknown hashtag link in', self.fn + ':', '#' + bad) def handle_UE(self): st = self.state if st.txt.startswith(('.', ',', '!', '?', ';', ':')): st.man_out[-1] = ".UE " + st.txt[0] + "\n" st.txt = st.txt[1:] st.after_a_tag = False def handle_starttag(self, tag, attrs_list): st = self.state if args.debug: self.output_debug('START', (tag, attrs_list)) if st.at_first_tag_in_li: if st.list_state[-1] == 'dl': st.dt_from = tag if tag == 'p': tag = 'dt' else: st.html_out.append('<dt>') elif tag == 'p': st.at_first_tag_in_dd = True # Kluge to suppress a .P at the start of an li. st.at_first_tag_in_li = False if tag == 'p': if not st.at_first_tag_in_dd: st.man_out.append(st.p_macro) elif tag == 'li': st.at_first_tag_in_li = True lstate = st.list_state[-1] if lstate == 'dl': return if lstate == 'o': st.man_out.append(".IP o\n") else: st.man_out.append(".IP " + str(lstate) + ".\n") st.list_state[-1] += 1 elif tag == 'blockquote': st.man_out.append(".RS 4\n") elif tag == 'pre': st.in_pre = True st.man_out.append(st.p_macro + ".nf\n") elif tag == 'code' and not st.in_pre: st.in_code = True st.txt += BOLD_FONT[0] elif tag == 'strong' or tag == 'b': st.txt += BOLD_FONT[0] elif tag == 'em' or tag == 'i': if st.want_manpage: tag = 'u' # Change it into underline to be more like the manpage st.txt += UNDR_FONT[0] elif tag == 'ol': start = 1 for var, val in attrs_list: if var == 'start': start = int(val) # We only support integers. break if st.list_state: st.man_out.append(".RS\n") if start == 0: tag = 'dl' attrs_list = [ ] st.list_state.append('dl') else: st.list_state.append(start) st.man_out.append(st.p_macro) st.p_macro = ".IP\n" elif tag == 'ul': st.man_out.append(st.p_macro) if st.list_state: st.man_out.append(".RS\n") st.p_macro = ".IP\n" st.list_state.append('o') elif tag == 'hr': st.man_out.append(".l\n") st.html_out.append("<hr />") return elif tag == 'a': st.a_href = None for var, val in attrs_list: if var == 'href': if val.startswith(('https://', 'http://', 'mailto:', 'ftp:')): if st.after_a_tag: self.handle_UE() st.man_out.append(manify(st.txt.strip()) + "\n") st.man_out.append(".UR " + val + "\n") st.txt = '' st.a_href = val st.a_href_external = True elif '#' in val: pg, tgt = val.split('#', 1) if pg and pg not in VALID_PAGES or '#' in tgt: st.bad_hashtags.add(val) elif tgt in ('', 'opt', 'dopt'): st.a_href = val st.a_href_external = False elif pg == '': st.referenced_hashtags.add(tgt) if tgt in st.latest_targets: warn('Found link to the current section in', self.fn + ':', val) elif val not in VALID_PAGES: st.bad_hashtags.add(val) st.a_txt_start = len(st.txt) st.html_out.append('<' + tag + ''.join(' ' + var + '="' + htmlify(val) + '"' for var, val in attrs_list) + '>') st.at_first_tag_in_dd = False def handle_endtag(self, tag): st = self.state if args.debug: self.output_debug('END', (tag,)) if st.after_a_tag: self.handle_UE() if tag in CONSUMES_TXT or st.dt_from == tag: txt = st.txt.strip() st.txt = '' else: txt = None add_to_txt = None if tag == 'h1': tgt = txt target_suf = '' if tgt.startswith('NEWS for '): m = VERSION_RE.search(tgt) if m: tgt = m.group(1) st.target_suf = '-' + tgt self.add_targets(tag, tgt) elif tag == 'h2': st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n') self.add_targets(tag, txt, st.target_suf) st.opt_prefix = 'dopt' if txt == 'DAEMON OPTIONS' else 'opt' elif tag == 'h3': st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n') self.add_targets(tag, txt, st.target_suf) elif tag == 'p': if st.dt_from == 'p': tag = 'dt' st.man_out.append('.IP "' + manify(txt) + '"\n') if txt.startswith(BOLD_FONT[0]): self.add_targets(tag, txt) st.dt_from = None elif txt != '': st.man_out.append(manify(txt) + "\n") elif tag == 'li': if st.list_state[-1] == 'dl': if st.at_first_tag_in_li: die("Invalid 0. -> td translation") tag = 'dd' if txt != '': st.man_out.append(manify(txt) + "\n") st.at_first_tag_in_li = False elif tag == 'blockquote': st.man_out.append(".RE\n") elif tag == 'pre': st.in_pre = False st.man_out.append(manify(txt) + "\n.fi\n") elif (tag == 'code' and not st.in_pre): st.in_code = False add_to_txt = NORM_FONT[0] elif tag == 'strong' or tag == 'b': add_to_txt = NORM_FONT[0] elif tag == 'em' or tag == 'i': if st.want_manpage: tag = 'u' # Change it into underline to be more like the manpage add_to_txt = NORM_FONT[0] elif tag == 'ol' or tag == 'ul': if st.list_state.pop() == 'dl': tag = 'dl' if st.list_state: st.man_out.append(".RE\n") else: st.p_macro = ".P\n" st.at_first_tag_in_dd = False elif tag == 'hr': return elif tag == 'a': if st.a_href_external: st.txt = st.txt.strip()