#!/usr/bin/env python3# SPDX-License-Identifier: GPL-2.0# Copyright (C) 2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org>## pylint: disable=R0902, R0912, R0913, R0914, R0915, R0917, C0103## Converted from docs Makefile and parallel-wrapper.sh, both under# GPLv2, copyrighted since 2008 by the following authors:##    Akira Yokosawa <akiyks@gmail.com>#    Arnd Bergmann <arnd@arndb.de>#    Breno Leitao <leitao@debian.org>#    Carlos Bilbao <carlos.bilbao@amd.com>#    Dave Young <dyoung@redhat.com>#    Donald Hunter <donald.hunter@gmail.com>#    Geert Uytterhoeven <geert+renesas@glider.be>#    Jani Nikula <jani.nikula@intel.com>#    Jan Stancek <jstancek@redhat.com>#    Jonathan Corbet <corbet@lwn.net>#    Joshua Clayton <stillcompiling@gmail.com>#    Kees Cook <keescook@chromium.org>#    Linus Torvalds <torvalds@linux-foundation.org>#    Magnus Damm <damm+renesas@opensource.se>#    Masahiro Yamada <masahiroy@kernel.org>#    Mauro Carvalho Chehab <mchehab+huawei@kernel.org>#    Maxim Cournoyer <maxim.cournoyer@gmail.com>#    Peter Foley <pefoley2@pefoley.com>#    Randy Dunlap <rdunlap@infradead.org>#    Rob Herring <robh@kernel.org>#    Shuah Khan <shuahkh@osg.samsung.com>#    Thorsten Blum <thorsten.blum@toblux.com>#    Tomas Winkler <tomas.winkler@intel.com>  """Sphinx build wrapper that handles Kernel-specific business rules: - it gets the Kernel build environment vars;- it determines what's the best parallelism;- it handles SPHINXDIRS This tool ensures that MIN_PYTHON_VERSION is satisfied. If version isbelow that, it seeks for a new Python version. If found, it re-runs usingthe newer version.""" import argparseimport localeimport osimport reimport shleximport shutilimport subprocessimport sys from concurrent import futuresfrom glob import glob  LIB_DIR = "../lib/python"SRC_DIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) from kdoc.python_version import PythonVersionfrom kdoc.latex_fonts import LatexFontCheckerfrom jobserver import JobserverExec         # pylint: disable=C0413,C0411,E0401 ##  Some constants#VENV_DEFAULT = "sphinx_latest"MIN_PYTHON_VERSION = PythonVersion("3.7").versionPAPER = ["", "a4", "letter"] TARGETS = {    "cleandocs":     { "builder": "clean" },    "linkcheckdocs": { "builder": "linkcheck" },    "htmldocs":      { "builder": "html" },    "epubdocs":      { "builder": "epub",    "out_dir": "epub" },    "texinfodocs":   { "builder": "texinfo", "out_dir": "texinfo" },    "infodocs":      { "builder": "texinfo", "out_dir": "texinfo" },    "mandocs":       { "builder": "man",     "out_dir": "man" },    "latexdocs":     { "builder": "latex",   "out_dir": "latex" },    "pdfdocs":       { "builder": "latex",   "out_dir": "latex" },    "xmldocs":       { "builder": "xml",     "out_dir": "xml" },}  ## SphinxBuilder class# class SphinxBuilder:    """    Handles a sphinx-build target, adding needed arguments to build    with the Kernel.    """     def get_path(self, path, use_cwd=False, abs_path=False):        """        Ancillary routine to handle patches the right way, as shell does.         It first expands "~" and "~user". Then, if patch is not absolute,        join self.srctree. Finally, if requested, convert to abspath.        """         path = os.path.expanduser(path)        if not path.startswith("/"):            if use_cwd:                base = os.getcwd()            else:                base = self.srctree             path = os.path.join(base, path)         if abs_path:            return os.path.abspath(path)         return path     def check_rust(self, sphinxdirs):        """        Checks if Rust is enabled        """        config = os.path.join(self.srctree, ".config")         if not {'.', 'rust'}.intersection(sphinxdirs):            return False         if not os.path.isfile(config):            return False         re_rust = re.compile(r"CONFIG_RUST=(m|y)")         try:            with open(config, "r", encoding="utf-8") as fp:                for line in fp:                    if re_rust.match(line):                        return True         except OSError as e:            print(f"Failed to open {config}", file=sys.stderr)            return False         return False     def get_sphinx_extra_opts(self, n_jobs):        """        Get the number of jobs to be used for docs build passed via command        line and desired sphinx verbosity.         The number of jobs can be on different places:         1) It can be passed via "-j" argument;        2) The SPHINXOPTS="-j8" env var may have "-j";        3) if called via GNU make, -j specifies the desired number of jobs.           with GNU makefile, this number is available via POSIX jobserver;        4) if none of the above is available, it should default to "-jauto",           and let sphinx decide the best value.        """         #        # SPHINXOPTS env var, if used, contains extra arguments to be used        # by sphinx-build time. Among them, it may contain sphinx verbosity        # and desired number of parallel jobs.        #        parser = argparse.ArgumentParser()        parser.add_argument('-j', '--jobs', type=int)        parser.add_argument('-q', '--quiet', action='store_true')        parser.add_argument('-v', '--verbose', default=0, action='count')         #        # Other sphinx-build arguments go as-is, so place them        # at self.sphinxopts, using shell parser        #        sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", ""))         #        # Build a list of sphinx args, honoring verbosity here if specified        #         sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts)         verbose = sphinx_args.verbose        if self.verbose:            verbose += 1         if sphinx_args.quiet is True:            verbose = 0         #        # If the user explicitly sets "-j" at command line, use it.        # Otherwise, pick it from SPHINXOPTS args        #        if n_jobs:            self.n_jobs = n_jobs        elif sphinx_args.jobs:            self.n_jobs = sphinx_args.jobs        else:            self.n_jobs = None         if verbose < 1:            self.sphinxopts += ["-q"]        else:            for i in range(1, sphinx_args.verbose):                self.sphinxopts += ["-v"]     def __init__(self, builddir, venv=None, verbose=False, n_jobs=None,                 interactive=None):        """Initialize internal variables"""        self.venv = venv        self.verbose = None         #        # Normal variables passed from Kernel's makefile        #        self.kernelversion = os.environ.get("KERNELVERSION", "unknown")        self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown")        self.pdflatex = os.environ.get("PDFLATEX", "xelatex")         #        # Kernel main Makefile defines a PYTHON3 variable whose default is        # "python3". When set to a different value, it allows running a        # diferent version than the default official python3 package.        # Several distros package python3xx-sphinx packages with newer        # versions of Python and sphinx-build.        #        # Honor such variable different than default        #        self.python = os.environ.get("PYTHON3")        if self.python == "python3":            self.python = None         if not interactive:            self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape")        else:            self.latexopts = os.environ.get("LATEXOPTS", "")         if not verbose:            try:                verbose = bool(int(os.environ.get("KBUILD_VERBOSE", 0)))            except ValueError:                # Handles an eventual case where verbosity is not a number                # like KBUILD_VERBOSE=""                verbose = False         if verbose is not None:            self.verbose = verbose         #        # Source tree directory. This needs to be at os.environ, as        # Sphinx extensions use it        #        self.srctree = os.environ.get("srctree")        if not self.srctree:            self.srctree = "."            os.environ["srctree"] = self.srctree         #        # Now that we can expand srctree, get other directories as well        #        self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build")        self.kerneldoc = self.get_path(os.environ.get("KERNELDOC",                                                      "tools/docs/kernel-doc"))        self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True)         #        # Get directory locations for LaTeX build toolchain        #        self.pdflatex_cmd = shutil.which(self.pdflatex)        self.latexmk_cmd = shutil.which("latexmk")         self.env = os.environ.copy()         self.get_sphinx_extra_opts(n_jobs)         #        # If venv command line argument is specified, run Sphinx from venv        #        if venv:            bin_dir = os.path.join(venv, "bin")            if not os.path.isfile(os.path.join(bin_dir, "activate")):                sys.exit(f"Venv {venv} not found.")             # "activate" virtual env            self.env["PATH"] = bin_dir + ":" + self.env["PATH"]            self.env["VIRTUAL_ENV"] = venv            if "PYTHONHOME" in self.env:                del self.env["PYTHONHOME"]            print(f"Setting venv to {venv}")     def run_sphinx(self, sphinx_build, build_args, *args, **pwargs):        """        Executes sphinx-build using current python3 command.         When calling via GNU make, POSIX jobserver is used to tell how        many jobs are still available from a job pool. claim all remaining        jobs, as we don't want sphinx-build to run in parallel with other        jobs.         Despite that, the user may actually force a different value than        the number of available jobs via command line.         The "with" logic here is used to ensure that the claimed jobs will        be freed once subprocess finishes        """         with JobserverExec() as jobserver:            if jobserver.claim:                #                # when GNU make is used, claim available jobs from jobserver                #                n_jobs = str(jobserver.claim)            else:                #                # Otherwise, let sphinx decide by default                #                n_jobs = "auto"             #            # If explicitly requested via command line, override default            #            if self.n_jobs:                n_jobs = str(self.n_jobs)             #            # We can't simply call python3 sphinx-build, as OpenSUSE            # Tumbleweed uses an ELF binary file (/usr/bin/alts) to switch            # between different versions of sphinx-build. So, only call it            # prepending "python3.xx" when PYTHON3 variable is not default.            #            if self.python:                cmd = [self.python]            else:                cmd = []             cmd += [sphinx_build]            cmd += [f"-j{n_jobs}"]            cmd += build_args            cmd += self.sphinxopts             if self.verbose:                print(" ".join(cmd))             return subprocess.call(cmd, *args, **pwargs)     def handle_html(self, css, output_dir):        """        Extra steps for HTML and epub output.         For such targets, we need to ensure that CSS will be properly        copied to the output _static directory        """         if css:            css = os.path.expanduser(css)            if not css.startswith("/"):                css = os.path.join(self.srctree, css)             static_dir = os.path.join(output_dir, "_static")            os.makedirs(static_dir, exist_ok=True)             try:                shutil.copy2(css, static_dir)            except (OSError, IOError) as e:                print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr)     def build_pdf_file(self, latex_cmd, from_dir, path):        """Builds a single pdf file using latex_cmd"""        try:            subprocess.run(latex_cmd + [path],                            cwd=from_dir, check=True, env=self.env)             return True        except subprocess.CalledProcessError:            return False     def pdf_parallel_build(self, tex_suffix, latex_cmd, tex_files, n_jobs):        """Build PDF files in parallel if possible"""        builds = {}        build_failed = False        max_len = 0        has_tex = False         #        # LaTeX PDF error code is almost useless for us:        # any warning makes it non-zero. For kernel doc builds it always return        # non-zero even when build succeeds. So, let's do the best next thing:        # Ignore build errors. At the end, check if all PDF files were built,        # printing a summary with the built ones and returning 0 if all of        # them were actually built.        #        with futures.ThreadPoolExecutor(max_workers=n_jobs) as executor:            jobs = {}             for from_dir, pdf_dir, entry in tex_files:                name = entry.name                 if not name.endswith(tex_suffix):                    continue                 name = name[:-len(tex_suffix)]                has_tex = True                 future = executor.submit(self.build_pdf_file, latex_cmd,                                         from_dir, entry.path)                jobs[future] = (from_dir, pdf_dir, name)             for future in futures.as_completed(jobs):                from_dir, pdf_dir, name = jobs[future]                 pdf_name = name + ".pdf"                pdf_from = os.path.join(from_dir, pdf_name)                pdf_to = os.path.join(pdf_dir, pdf_name)                out_name = os.path.relpath(pdf_to, self.builddir)                max_len = max(max_len, len(out_name))                 try:                    success = future.result()                     if success and os.path.exists(pdf_from):                        os.rename(pdf_from, pdf_to)                         #                        # if verbose, get the name of built PDF file                        #                        if self.verbose:                           builds[out_name] = "SUCCESS"                    else:                        builds[out_name] = "FAILED"                        build_failed = True                except futures.Error as e:                    builds[out_name] = f"FAILED ({repr(e)})"                    build_failed = True         #        # Handle case where no .tex files were found        #        if not has_tex:            out_name = "LaTeX files"            max_len = max(max_len, len(out_name))            builds[out_name] = "FAILED: no .tex files were generated"            build_failed = True         return builds, build_failed, max_len     def handle_pdf(self, output_dirs, deny_vf):        """        Extra steps for PDF output.         As PDF is handled via a LaTeX output, after building the .tex file,        a new build is needed to create the PDF output from the latex        directory.        """        builds = {}        max_len = 0        tex_suffix = ".tex"        tex_files = []         #        # Since early 2024, Fedora and openSUSE tumbleweed have started        # deploying variable-font format of "Noto CJK", causing LaTeX        # to break with CJK. Work around it, by denying the variable font        # usage during xelatex build by passing the location of a config        # file with a deny list.        #        # See tools/docs/lib/latex_fonts.py for more details.        #        if deny_vf:            deny_vf = os.path.expanduser(deny_vf)            if os.path.isdir(deny_vf):                self.env["XDG_CONFIG_HOME"] = deny_vf         for from_dir in output_dirs:            pdf_dir = os.path.join(from_dir, "../pdf")            os.makedirs(pdf_dir, exist_ok=True)             if self.latexmk_cmd:                latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"]            else:                latex_cmd = [self.pdflatex]             latex_cmd.extend(shlex.split(self.latexopts))             # Get a list of tex files to process            with os.scandir(from_dir) as it:                for entry in it:                    if entry.name.endswith(tex_suffix):                        tex_files.append((from_dir, pdf_dir, entry))         #        # When using make, this won't be used, as the number of jobs comes        # from POSIX jobserver. So, this covers the case where build comes        # from command line. On such case, serialize by default, except if        # the user explicitly sets the number of jobs.        #        n_jobs = 1         # n_jobs is either an integer or "auto". Only use it if it is a number

Lines 1–500 of 902. Load more