#!/usr/bin/env python3# SPDX-License-Identifier: GPL-2.0-or-later# Copyright (c) 2017-2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org>## pylint: disable=C0103,C0114,C0115,C0116,C0301,C0302# pylint: disable=R0902,R0904,R0911,R0912,R0914,R0915,R1705,R1710,E1121 # Note: this script requires at least Python 3.6 to run.# Don't add changes not compatible with it, it is meant to report# incompatible python versions. """Dependency checker for Sphinx documentation Kernel build. This module provides tools to check for all required dependencies needed tobuild documentation using Sphinx, including system packages, Python modulesand LaTeX packages for PDF generation. It detect packages for a subset of Linux distributions used by Kernelmaintainers, showing hints and missing dependencies. The main class SphinxDependencyChecker handles the dependency checking logicand provides recommendations for installing missing packages. It supports bothsystem package installations and  Python virtual environments. By default,system pacage install is recommended.""" import argparseimport osimport reimport subprocessimport sysfrom glob import glob  def parse_version(version):    """Convert a major.minor.patch version into a tuple"""    return tuple(int(x) for x in version.split("."))  def ver_str(version):    """Returns a version tuple as major.minor.patch"""     return ".".join([str(x) for x in version])  RECOMMENDED_VERSION = parse_version("3.4.3")MIN_PYTHON_VERSION = parse_version("3.7")  class DepManager:    """    Manage package dependencies. There are three types of dependencies:     - System: dependencies required for docs build;    - Python: python dependencies for a native distro Sphinx install;    - PDF: dependencies needed by PDF builds.     Each dependency can be mandatory or optional. Not installing an optional    dependency won't break the build, but will cause degradation at the    docs output.    """     # Internal types of dependencies. Don't use them outside DepManager class.    _SYS_TYPE = 0    _PHY_TYPE = 1    _PDF_TYPE = 2     # Dependencies visible outside the class.    # The keys are tuple with: (type, is_mandatory flag).    #    # Currently we're not using all optional dep types. Yet, we'll keep all    # possible combinations here. They're not many, and that makes easier    # if later needed and for the name() method below     SYSTEM_MANDATORY = (_SYS_TYPE, True)    PYTHON_MANDATORY = (_PHY_TYPE, True)    PDF_MANDATORY = (_PDF_TYPE, True)     SYSTEM_OPTIONAL = (_SYS_TYPE, False)    PYTHON_OPTIONAL = (_PHY_TYPE, False)    PDF_OPTIONAL = (_PDF_TYPE, True)     def __init__(self, pdf):        """        Initialize internal vars:         - missing: missing dependencies list, containing a distro-independent                   name for a missing dependency and its type.        - missing_pkg: ancillary dict containing missing dependencies in                       distro namespace, organized by type.        - need: total number of needed dependencies. Never cleaned.        - optional: total number of optional dependencies. Never cleaned.        - pdf: Is PDF support enabled?        """        self.missing = {}        self.missing_pkg = {}        self.need = 0        self.optional = 0        self.pdf = pdf     @staticmethod    def name(dtype):        """        Ancillary routine to output a warn/error message reporting        missing dependencies.        """        if dtype[0] == DepManager._SYS_TYPE:            msg = "build"        elif dtype[0] == DepManager._PHY_TYPE:            msg = "Python"        else:            msg = "PDF"         if dtype[1]:            return f"ERROR: {msg} mandatory deps missing"        else:            return f"Warning: {msg} optional deps missing"     @staticmethod    def is_optional(dtype):        """Ancillary routine to report if a dependency is optional"""        return not dtype[1]     @staticmethod    def is_pdf(dtype):        """Ancillary routine to report if a dependency is for PDF generation"""        if dtype[0] == DepManager._PDF_TYPE:            return True         return False     def add_package(self, package, dtype):        """        Add a package at the self.missing() dictionary.        Doesn't update missing_pkg.        """        is_optional = DepManager.is_optional(dtype)        self.missing[package] = dtype        if is_optional:            self.optional += 1        else:            self.need += 1     def del_package(self, package):        """        Remove a package at the self.missing() dictionary.        Doesn't update missing_pkg.        """        if package in self.missing:            del self.missing[package]     def clear_deps(self):        """        Clear dependencies without changing needed/optional.         This is an ackward way to have a separate section to recommend        a package after system main dependencies.         TODO: rework the logic to prevent needing it.        """         self.missing = {}        self.missing_pkg = {}     def check_missing(self, progs):        """        Update self.missing_pkg, using progs dict to convert from the        agnostic package name to distro-specific one.         Returns an string with the packages to be installed, sorted and        with eventual duplicates removed.        """         self.missing_pkg = {}         for prog, dtype in sorted(self.missing.items()):            # At least on some LTS distros like CentOS 7, texlive doesn't            # provide all packages we need. When such distros are            # detected, we have to disable PDF output.            #            # So, we need to ignore the packages that distros would            # need for LaTeX to work            if DepManager.is_pdf(dtype) and not self.pdf:                self.optional -= 1                continue             if not dtype in self.missing_pkg:                self.missing_pkg[dtype] = []             self.missing_pkg[dtype].append(progs.get(prog, prog))         install = []        for dtype, pkgs in self.missing_pkg.items():            install += pkgs         return " ".join(sorted(set(install)))     def warn_install(self):        """        Emit warnings/errors related to missing packages.        """         output_msg = ""         for dtype in sorted(self.missing_pkg.keys()):            progs = " ".join(sorted(set(self.missing_pkg[dtype])))             try:                name = DepManager.name(dtype)                output_msg += f'{name}:\t{progs}\n'            except KeyError:                raise KeyError(f"ERROR!!!: invalid dtype for {progs}: {dtype}")         if output_msg:            print(f"\n{output_msg}") class AncillaryMethods:    """    Ancillary methods that checks for missing dependencies for different    types of types, like binaries, python modules, rpm deps, etc.    """     @staticmethod    def which(prog):        """        Our own implementation of which(). We could instead use        shutil.which(), but this function is simple enough.        Probably faster to use this implementation than to import shutil.        """        for path in os.environ.get("PATH", "").split(":"):            full_path = os.path.join(path, prog)            if os.access(full_path, os.X_OK):                return full_path         return None     @staticmethod    def get_python_version(cmd):        """        Get python version from a Python binary. As we need to detect if        are out there newer python binaries, we can't rely on sys.release here.        """         result = SphinxDependencyChecker.run([cmd, "--version"],                                            capture_output=True, text=True)        version = result.stdout.strip()         match = re.search(r"(\d+\.\d+\.\d+)", version)        if match:            return parse_version(match.group(1))         print(f"Can't parse version {version}")        return (0, 0, 0)     @staticmethod    def find_python():        """        Detect if are out there any python 3.xy version newer than the        current one.         Note: this routine is limited to up to 2 digits for python3. We        may need to update it one day, hopefully on a distant future.        """        patterns = [            "python3.[0-9]",            "python3.[0-9][0-9]",        ]         # Seek for a python binary newer than MIN_PYTHON_VERSION        for path in os.getenv("PATH", "").split(":"):            for pattern in patterns:                for cmd in glob(os.path.join(path, pattern)):                    if os.path.isfile(cmd) and os.access(cmd, os.X_OK):                        version = SphinxDependencyChecker.get_python_version(cmd)                        if version >= MIN_PYTHON_VERSION:                            return cmd     @staticmethod    def check_python():        """        Check if the current python binary satisfies our minimal requirement        for Sphinx build. If not, re-run with a newer version if found.        """        cur_ver = sys.version_info[:3]        if cur_ver >= MIN_PYTHON_VERSION:            ver = ver_str(cur_ver)            print(f"Python version: {ver}")             # This could be useful for debugging purposes            if SphinxDependencyChecker.which("docutils"):                result = SphinxDependencyChecker.run(["docutils", "--version"],                                                    capture_output=True, text=True)                ver = result.stdout.strip()                match = re.search(r"(\d+\.\d+\.\d+)", ver)                if match:                    ver = match.group(1)                 print(f"Docutils version: {ver}")             return         python_ver = ver_str(cur_ver)         new_python_cmd = SphinxDependencyChecker.find_python()        if not new_python_cmd:            print(f"ERROR: Python version {python_ver} is not spported anymore\n")            print("       Can't find a new version. This script may fail")            return         # Restart script using the newer version        script_path = os.path.abspath(sys.argv[0])        args = [new_python_cmd, script_path] + sys.argv[1:]         print(f"Python {python_ver} not supported. Changing to {new_python_cmd}")         try:            os.execv(new_python_cmd, args)        except OSError as e:            sys.exit(f"Failed to restart with {new_python_cmd}: {e}")     @staticmethod    def run(*args, **kwargs):        """        Excecute a command, hiding its output by default.        Preserve comatibility with older Python versions.        """         capture_output = kwargs.pop('capture_output', False)         if capture_output:            if 'stdout' not in kwargs:                kwargs['stdout'] = subprocess.PIPE            if 'stderr' not in kwargs:                kwargs['stderr'] = subprocess.PIPE        else:            if 'stdout' not in kwargs:                kwargs['stdout'] = subprocess.DEVNULL            if 'stderr' not in kwargs:                kwargs['stderr'] = subprocess.DEVNULL         # Don't break with older Python versions        if 'text' in kwargs and sys.version_info < (3, 7):            kwargs['universal_newlines'] = kwargs.pop('text')         return subprocess.run(*args, **kwargs) class MissingCheckers(AncillaryMethods):    """    Contains some ancillary checkers for different types of binaries and    package managers.    """     def __init__(self, args, texlive):        """        Initialize its internal variables        """        self.pdf = args.pdf        self.virtualenv = args.virtualenv        self.version_check = args.version_check        self.texlive = texlive         self.min_version = (0, 0, 0)        self.cur_version = (0, 0, 0)         self.deps = DepManager(self.pdf)         self.need_symlink = 0        self.need_sphinx = 0         self.verbose_warn_install = 1         self.virtenv_dir = ""        self.install = ""        self.python_cmd = ""         self.virtenv_prefix = ["sphinx_", "Sphinx_" ]     def check_missing_file(self, files, package, dtype):        """        Does the file exists? If not, add it to missing dependencies.        """        for f in files:            if os.path.exists(f):                return        self.deps.add_package(package, dtype)     def check_program(self, prog, dtype):        """        Does the program exists and it is at the PATH?        If not, add it to missing dependencies.        """        found = self.which(prog)        if found:            return found         self.deps.add_package(prog, dtype)         return None     def check_perl_module(self, prog, dtype):        """        Does perl have a dependency? Is it available?        If not, add it to missing dependencies.         Right now, we still need Perl for doc build, as it is required        by some tools called at docs or kernel build time, like:             scripts/documentation-file-ref-check         Also, checkpatch is on Perl.        """         # While testing with lxc download template, one of the        # distros (Oracle) didn't have perl - nor even an option to install        # before installing oraclelinux-release-el9 package.        #        # Check it before running an error. If perl is not there,        # add it as a mandatory package, as some parts of the doc builder        # needs it.        if not self.which("perl"):            self.deps.add_package("perl", DepManager.SYSTEM_MANDATORY)            self.deps.add_package(prog, dtype)            return         try:            self.run(["perl", f"-M{prog}", "-e", "1"], check=True)        except subprocess.CalledProcessError:            self.deps.add_package(prog, dtype)     def check_python_module(self, module, is_optional=False):        """        Does a python module exists outside venv? If not, add it to missing        dependencies.        """        if is_optional:            dtype = DepManager.PYTHON_OPTIONAL        else:            dtype = DepManager.PYTHON_MANDATORY         try:            self.run([self.python_cmd, "-c", f"import {module}"], check=True)        except subprocess.CalledProcessError:            self.deps.add_package(module, dtype)     def check_rpm_missing(self, pkgs, dtype):        """        Does a rpm package exists? If not, add it to missing dependencies.        """        for prog in pkgs:            try:                self.run(["rpm", "-q", prog], check=True)            except subprocess.CalledProcessError:                self.deps.add_package(prog, dtype)     def check_pacman_missing(self, pkgs, dtype):        """        Does a pacman package exists? If not, add it to missing dependencies.        """        for prog in pkgs:            try:                self.run(["pacman", "-Q", prog], check=True)            except subprocess.CalledProcessError:                self.deps.add_package(prog, dtype)     def check_missing_tex(self, is_optional=False):        """        Does a LaTeX package exists? If not, add it to missing dependencies.        """        if is_optional:            dtype = DepManager.PDF_OPTIONAL        else:            dtype = DepManager.PDF_MANDATORY         kpsewhich = self.which("kpsewhich")        for prog, package in self.texlive.items():             # If kpsewhich is not there, just add it to deps            if not kpsewhich:                self.deps.add_package(package, dtype)                continue             # Check if the package is needed            try:                result = self.run(                    [kpsewhich, prog], stdout=subprocess.PIPE, text=True, check=True                )                 # Didn't find. Add it                if not result.stdout.strip():                    self.deps.add_package(package, dtype)             except subprocess.CalledProcessError:                # kpsewhich returned an error. Add it, just in case                self.deps.add_package(package, dtype)     def get_sphinx_fname(self):        """        Gets the binary filename for sphinx-build.        """

Lines 1–500 of 1621. Load more