#!/usr/bin/env python3# SPDX-License-Identifier: GPL-2.0-or-later# Copyright (c) 2017-2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org>## pylint: disable=C0103,C0114,C0115,C0116,C0301,C0302# pylint: disable=R0902,R0904,R0911,R0912,R0914,R0915,R1705,R1710,E1121 # Note: this script requires at least Python 3.6 to run.# Don't add changes not compatible with it, it is meant to report# incompatible python versions. """Dependency checker for Sphinx documentation Kernel build. This module provides tools to check for all required dependencies needed tobuild documentation using Sphinx, including system packages, Python modulesand LaTeX packages for PDF generation. It detect packages for a subset of Linux distributions used by Kernelmaintainers, showing hints and missing dependencies. The main class SphinxDependencyChecker handles the dependency checking logicand provides recommendations for installing missing packages. It supports bothsystem package installations and  Python virtual environments. By default,system pacage install is recommended.""" import argparseimport localeimport osimport reimport subprocessimport sysfrom glob import globimport os.path src_dir = os.path.dirname(os.path.realpath(__file__))sys.path.insert(0, os.path.join(src_dir, '../lib/python'))from kdoc.python_version import PythonVersion RECOMMENDED_VERSION = PythonVersion("3.4.3").versionMIN_PYTHON_VERSION = PythonVersion("3.7").version  class DepManager:    """    Manage package dependencies. There are three types of dependencies:     - System: dependencies required for docs build;    - Python: python dependencies for a native distro Sphinx install;    - PDF: dependencies needed by PDF builds.     Each dependency can be mandatory or optional. Not installing an optional    dependency won't break the build, but will cause degradation at the    docs output.    """     # Internal types of dependencies. Don't use them outside DepManager class.    _SYS_TYPE = 0    _PHY_TYPE = 1    _PDF_TYPE = 2     # Dependencies visible outside the class.    # The keys are tuple with: (type, is_mandatory flag).    #    # Currently we're not using all optional dep types. Yet, we'll keep all    # possible combinations here. They're not many, and that makes easier    # if later needed and for the name() method below     SYSTEM_MANDATORY = (_SYS_TYPE, True)    PYTHON_MANDATORY = (_PHY_TYPE, True)    PDF_MANDATORY = (_PDF_TYPE, True)     SYSTEM_OPTIONAL = (_SYS_TYPE, False)    PYTHON_OPTIONAL = (_PHY_TYPE, False)    PDF_OPTIONAL = (_PDF_TYPE, True)     def __init__(self, pdf):        """        Initialize internal vars:         - missing: missing dependencies list, containing a distro-independent                   name for a missing dependency and its type.        - missing_pkg: ancillary dict containing missing dependencies in                       distro namespace, organized by type.        - need: total number of needed dependencies. Never cleaned.        - optional: total number of optional dependencies. Never cleaned.        - pdf: Is PDF support enabled?        """        self.missing = {}        self.missing_pkg = {}        self.need = 0        self.optional = 0        self.pdf = pdf     @staticmethod    def name(dtype):        """        Ancillary routine to output a warn/error message reporting        missing dependencies.        """        if dtype[0] == DepManager._SYS_TYPE:            msg = "build"        elif dtype[0] == DepManager._PHY_TYPE:            msg = "Python"        else:            msg = "PDF"         if dtype[1]:            return f"ERROR: {msg} mandatory deps missing"        else:            return f"Warning: {msg} optional deps missing"     @staticmethod    def is_optional(dtype):        """Ancillary routine to report if a dependency is optional"""        return not dtype[1]     @staticmethod    def is_pdf(dtype):        """Ancillary routine to report if a dependency is for PDF generation"""        if dtype[0] == DepManager._PDF_TYPE:            return True         return False     def add_package(self, package, dtype):        """        Add a package at the self.missing() dictionary.        Doesn't update missing_pkg.        """        is_optional = DepManager.is_optional(dtype)        self.missing[package] = dtype        if is_optional:            self.optional += 1        else:            self.need += 1     def del_package(self, package):        """        Remove a package at the self.missing() dictionary.        Doesn't update missing_pkg.        """        if package in self.missing:            del self.missing[package]     def clear_deps(self):        """        Clear dependencies without changing needed/optional.         This is an ackward way to have a separate section to recommend        a package after system main dependencies.         TODO: rework the logic to prevent needing it.        """         self.missing = {}        self.missing_pkg = {}     def check_missing(self, progs):        """        Update self.missing_pkg, using progs dict to convert from the        agnostic package name to distro-specific one.         Returns an string with the packages to be installed, sorted and        with eventual duplicates removed.        """         self.missing_pkg = {}         for prog, dtype in sorted(self.missing.items()):            # At least on some LTS distros like CentOS 7, texlive doesn't            # provide all packages we need. When such distros are            # detected, we have to disable PDF output.            #            # So, we need to ignore the packages that distros would            # need for LaTeX to work            if DepManager.is_pdf(dtype) and not self.pdf:                self.optional -= 1                continue             if not dtype in self.missing_pkg:                self.missing_pkg[dtype] = []             self.missing_pkg[dtype].append(progs.get(prog, prog))         install = []        for dtype, pkgs in self.missing_pkg.items():            install += pkgs         return " ".join(sorted(set(install)))     def warn_install(self):        """        Emit warnings/errors related to missing packages.        """         output_msg = ""         for dtype in sorted(self.missing_pkg.keys()):            progs = " ".join(sorted(set(self.missing_pkg[dtype])))             try:                name = DepManager.name(dtype)                output_msg += f'{name}:\t{progs}\n'            except KeyError:                raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}")         if output_msg:            print(f"\n{output_msg}") class AncillaryMethods:    """    Ancillary methods that checks for missing dependencies for different    types of types, like binaries, python modules, rpm deps, etc.    """     @staticmethod    def which(prog):        """        Our own implementation of which(). We could instead use        shutil.which(), but this function is simple enough.        Probably faster to use this implementation than to import shutil.        """        for path in os.environ.get("PATH", "").split(":"):            full_path = os.path.join(path, prog)            if os.access(full_path, os.X_OK):                return full_path         return None     @staticmethod    def run(*args, **kwargs):        """        Excecute a command, hiding its output by default.        Preserve compatibility with older Python versions.        """         capture_output = kwargs.pop('capture_output', False)         if capture_output:            if 'stdout' not in kwargs:                kwargs['stdout'] = subprocess.PIPE            if 'stderr' not in kwargs:                kwargs['stderr'] = subprocess.PIPE        else:            if 'stdout' not in kwargs:                kwargs['stdout'] = subprocess.DEVNULL            if 'stderr' not in kwargs:                kwargs['stderr'] = subprocess.DEVNULL         # Don't break with older Python versions        if 'text' in kwargs and sys.version_info < (3, 7):            kwargs['universal_newlines'] = kwargs.pop('text')         return subprocess.run(*args, **kwargs) class MissingCheckers(AncillaryMethods):    """    Contains some ancillary checkers for different types of binaries and    package managers.    """     def __init__(self, args, texlive):        """        Initialize its internal variables        """        self.pdf = args.pdf        self.virtualenv = args.virtualenv        self.version_check = args.version_check        self.texlive = texlive         self.min_version = (0, 0, 0)        self.cur_version = (0, 0, 0)         self.deps = DepManager(self.pdf)         self.need_symlink = 0        self.need_sphinx = 0         self.verbose_warn_install = 1         self.virtenv_dir = ""        self.install = ""        self.python_cmd = ""         self.virtenv_prefix = ["sphinx_", "Sphinx_" ]     def check_missing_file(self, files, package, dtype):        """        Does the file exists? If not, add it to missing dependencies.        """        for f in files:            if os.path.exists(f):                return        self.deps.add_package(package, dtype)     def check_program(self, prog, dtype):        """        Does the program exists and it is at the PATH?        If not, add it to missing dependencies.        """        found = self.which(prog)        if found:            return found         self.deps.add_package(prog, dtype)         return None     def check_perl_module(self, prog, dtype):        """        Does perl have a dependency? Is it available?        If not, add it to missing dependencies.         Right now, we still need Perl for doc build, as it is required        by some tools called at docs or kernel build time, like:             tools/docs/documentation-file-ref-check         Also, checkpatch is on Perl.        """         # While testing with lxc download template, one of the        # distros (Oracle) didn't have perl - nor even an option to install        # before installing oraclelinux-release-el9 package.        #        # Check it before running an error. If perl is not there,        # add it as a mandatory package, as some parts of the doc builder        # needs it.        if not self.which("perl"):            self.deps.add_package("perl", DepManager.SYSTEM_MANDATORY)            self.deps.add_package(prog, dtype)            return         try:            self.run(["perl", f"-M{prog}", "-e", "1"], check=True)        except subprocess.CalledProcessError:            self.deps.add_package(prog, dtype)     def check_python_module(self, module, is_optional=False):        """        Does a python module exists outside venv? If not, add it to missing        dependencies.        """        if is_optional:            dtype = DepManager.PYTHON_OPTIONAL        else:            dtype = DepManager.PYTHON_MANDATORY         try:            self.run([self.python_cmd, "-c", f"import {module}"], check=True)        except subprocess.CalledProcessError:            self.deps.add_package(module, dtype)     def check_rpm_missing(self, pkgs, dtype):        """        Does a rpm package exists? If not, add it to missing dependencies.        """        for prog in pkgs:            try:                self.run(["rpm", "-q", prog], check=True)            except subprocess.CalledProcessError:                self.deps.add_package(prog, dtype)     def check_pacman_missing(self, pkgs, dtype):        """        Does a pacman package exists? If not, add it to missing dependencies.        """        for prog in pkgs:            try:                self.run(["pacman", "-Q", prog], check=True)            except subprocess.CalledProcessError:                self.deps.add_package(prog, dtype)     def check_missing_tex(self, is_optional=False):        """        Does a LaTeX package exists? If not, add it to missing dependencies.        """        if is_optional:            dtype = DepManager.PDF_OPTIONAL        else:            dtype = DepManager.PDF_MANDATORY         kpsewhich = self.which("kpsewhich")        for prog, package in self.texlive.items():             # If kpsewhich is not there, just add it to deps            if not kpsewhich:                self.deps.add_package(package, dtype)                continue             # Check if the package is needed            try:                result = self.run(                    [kpsewhich, prog], stdout=subprocess.PIPE, text=True, check=True                )                 # Didn't find. Add it                if not result.stdout.strip():                    self.deps.add_package(package, dtype)             except subprocess.CalledProcessError:                # kpsewhich returned an error. Add it, just in case                self.deps.add_package(package, dtype)     def get_sphinx_fname(self):        """        Gets the binary filename for sphinx-build.        """        if "SPHINXBUILD" in os.environ:            return os.environ["SPHINXBUILD"]         fname = "sphinx-build"        if self.which(fname):            return fname         fname = "sphinx-build-3"        if self.which(fname):            self.need_symlink = 1            return fname         return ""     def get_sphinx_version(self, cmd):        """        Gets sphinx-build version.        """        env = os.environ.copy()         # The sphinx-build tool has a bug: internally, it tries to set        # locale with locale.setlocale(locale.LC_ALL, ''). This causes a        # crash if language is not set. Detect and fix it.        try:            locale.setlocale(locale.LC_ALL, '')        except Exception:            env["LC_ALL"] = "C"            env["LANG"] = "C"         try:            result = self.run([cmd, "--version"], env=env,                              stdout=subprocess.PIPE,                              stderr=subprocess.STDOUT,                              text=True, check=True)        except (subprocess.CalledProcessError, FileNotFoundError):            return None         for line in result.stdout.split("\n"):            match = re.match(r"^sphinx-build\s+([\d\.]+)(?:\+(?:/[\da-f]+)|b\d+)?\s*$", line)            if match:                return PythonVersion.parse_version(match.group(1))             match = re.match(r"^Sphinx.*\s+([\d\.]+)\s*$", line)            if match:                return PythonVersion.parse_version(match.group(1))     def check_sphinx(self, conf):        """        Checks Sphinx minimal requirements        """        try:            with open(conf, "r", encoding="utf-8") as f:                for line in f:                    match = re.match(r"^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]", line)                    if match:                        self.min_version = PythonVersion.parse_version(match.group(1))                        break        except IOError:            sys.exit(f"Can't open {conf}")         if not self.min_version:            sys.exit(f"Can't get needs_sphinx version from {conf}")         self.virtenv_dir = self.virtenv_prefix[0] + "latest"         sphinx = self.get_sphinx_fname()        if not sphinx:            self.need_sphinx = 1            return         self.cur_version = self.get_sphinx_version(sphinx)        if not self.cur_version:            sys.exit(f"{sphinx} didn't return its version")         if self.cur_version < self.min_version:            curver = PythonVersion.ver_str(self.cur_version)            minver = PythonVersion.ver_str(self.min_version)             print(f"ERROR: Sphinx version is {curver}. It should be >= {minver}")            self.need_sphinx = 1            return         # On version check mode, just assume Sphinx has all mandatory deps        if self.version_check and self.cur_version >= RECOMMENDED_VERSION:            sys.exit(0)     def catcheck(self, filename):        """        Reads a file if it exists, returning as string.        If not found, returns an empty string.

Lines 1–500 of 1543. Load more