#!/usr/bin/env python3# SPDX-License-Identifier: GPL-2.0# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>.## pylint: disable=C0301,C0302,R0904,R0912,R0913,R0914,R0915,R0917,R1702 """Classes and functions related to reading a C language source or header FILEand extract embedded documentation comments from it.""" import sysimport refrom pprint import pformat from kdoc.c_lex import CTokenizer, tokenizer_set_logfrom kdoc.kdoc_re import KernRefrom kdoc.kdoc_item import KdocItem ## Regular expressions used to parse kernel-doc markups at KernelDoc class.## Let's declare them in lowercase outside any class to make it easier to# convert from the Perl script.## As those are evaluated at the beginning, no need to cache them# # Allow whitespace at end of comment start.doc_start = KernRe(r'^/\*\*\s*$', cache=False) doc_end = KernRe(r'\*/', cache=False)doc_com = KernRe(r'\s*\*\s*', cache=False)doc_com_body = KernRe(r'\s*\* ?', cache=False)doc_decl = doc_com + KernRe(r'(\w+)', cache=False) # @params and a strictly limited set of supported section names# Specifically:#   Match @word:#         @...:#         @{section-name}:# while trying to not match literal block starts like "example::"#known_section_names = 'description|context|returns?|notes?|examples?'known_sections = KernRe(known_section_names, flags = re.I)doc_sect = doc_com + \    KernRe(r'\s*(@[.\w]+|@\.\.\.|' + known_section_names + r')\s*:([^:].*)?$',           flags=re.I, cache=False) doc_content = doc_com_body + KernRe(r'(.*)', cache=False)doc_inline_start = KernRe(r'^\s*/\*\*\s*$', cache=False)doc_inline_sect = KernRe(r'\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)', cache=False)doc_inline_end = KernRe(r'^\s*\*/\s*$', cache=False)doc_inline_oneline = KernRe(r'^\s*/\*\*\s*(@\s*[\w][\w\.]*\s*):\s*(.*)\s*\*/\s*$', cache=False) export_symbol = KernRe(r'^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*', cache=False)export_symbol_ns = KernRe(r'^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*"\S+"\)\s*', cache=False) type_param = KernRe(r"@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) ## Tests for the beginning of a kerneldoc block in its various forms.#doc_block = doc_com + KernRe(r'DOC:\s*(.*)?', cache=False)doc_begin_data = KernRe(r"^\s*\*?\s*(struct|union|enum|typedef|var)\b\s*(\w*)", cache = False)doc_begin_func = KernRe(str(doc_com) +			# initial " * '                        r"(?:\w+\s*\*\s*)?" + 		# type (not captured)                        r'(?:define\s+)?' + 		# possible "define" (not captured)                        r'(\w+)\s*(?:\(\w*\))?\s*' +	# name and optional "(...)"                        r'(?:[-:].*)?$',		# description (not captured)                        cache = False) ## Ancillary functions# multi_space = KernRe(r'\s\s+')def trim_whitespace(s):    """    A little helper to get rid of excess white space.    """    return multi_space.sub(' ', s.strip()) def trim_private_members(text):    """    Remove ``struct``/``enum`` members that have been marked "private".    """     tokens = CTokenizer(text)    return str(tokens) class state:    """    States used by the parser's state machine.    """     # Parser states    NORMAL        = 0        #: Normal code.    NAME          = 1        #: Looking for function name.    DECLARATION   = 2        #: We have seen a declaration which might not be done.    BODY          = 3        #: The body of the comment.    SPECIAL_SECTION = 4      #: Doc section ending with a blank line.    PROTO         = 5        #: Scanning prototype.    DOCBLOCK      = 6        #: Documentation block.    INLINE_NAME   = 7        #: Gathering doc outside main block.    INLINE_TEXT   = 8	     #: Reading the body of inline docs.     #: Names for each parser state.    name = [        "NORMAL",        "NAME",        "DECLARATION",        "BODY",        "SPECIAL_SECTION",        "PROTO",        "DOCBLOCK",        "INLINE_NAME",        "INLINE_TEXT",    ]  SECTION_DEFAULT = "Description"  #: Default section. class KernelEntry:    """    Encapsulates a Kernel documentation entry.    """     def __init__(self, config, fname, ln):        self.config = config        self.fname = fname         self._contents = []        self.prototype = ""         self.warnings = []         self.parameterlist = []        self.parameterdescs = {}        self.parametertypes = {}        self.parameterdesc_start_lines = {}         self.sections_start_lines = {}        self.sections = {}         self.anon_struct_union = False         self.leading_space = None         self.fname = fname         # State flags        self.brcount = 0        self.declaration_start_line = ln + 1     #    # Management of section contents    #    def add_text(self, text):        """Add a new text to the entry contents list."""        self._contents.append(text)     def contents(self):        """Returns a string with all content texts that were added."""        return '\n'.join(self._contents) + '\n'     # TODO: rename to emit_message after removal of kernel-doc.pl    def emit_msg(self, ln, msg, *, warning=True):        """Emit a message."""         log_msg = f"{self.fname}:{ln} {msg}"         if not warning:            self.config.log.info(log_msg)            return         # Delegate warning output to output logic, as this way it        # will report warnings/info only for symbols that are output         self.warnings.append(log_msg)        return     def begin_section(self, line_no, title = SECTION_DEFAULT, dump = False):        """        Begin a new section.        """        if dump:            self.dump_section(start_new = True)        self.section = title        self.new_start_line = line_no     def dump_section(self, start_new=True):        """        Dumps section contents to arrays/hashes intended for that purpose.        """        #        # If we have accumulated no contents in the default ("description")        # section, don't bother.        #        if self.section == SECTION_DEFAULT and not self._contents:            return        name = self.section        contents = self.contents()         if type_param.match(name):            name = type_param.group(1)             self.parameterdescs[name] = contents            self.parameterdesc_start_lines[name] = self.new_start_line             self.new_start_line = 0         else:            if name in self.sections and self.sections[name] != "":                # Only warn on user-specified duplicate section names                if name != SECTION_DEFAULT:                    self.emit_msg(self.new_start_line,                                  f"duplicate section name '{name}'")                # Treat as a new paragraph - add a blank line                self.sections[name] += '\n' + contents            else:                self.sections[name] = contents                self.sections_start_lines[name] = self.new_start_line                self.new_start_line = 0 #        self.config.log.debug("Section: %s : %s", name, pformat(vars(self)))         if start_new:            self.section = SECTION_DEFAULT            self._contents = [] python_warning = False class KernelDoc:    """    Read a C language source or header FILE and extract embedded    documentation comments.    """     #: Name of context section.    section_context = "Context"     #: Name of return section.    section_return = "Return"     #: String to write when a parameter is not described.    undescribed = "-- undescribed --"     def __init__(self, config, fname, xforms, store_src=False):        """Initialize internal variables"""         self.fname = fname        self.config = config        self.xforms = xforms        self.store_src = store_src         tokenizer_set_log(self.config.log, f"{self.fname}: CMatch: ")         # Initial state for the state machines        self.state = state.NORMAL         # Store entry currently being processed        self.entry = None         # Place all potential outputs into an array        self.entries = []         #        # We need Python 3.7 for its "dicts remember the insertion        # order" guarantee        #        global python_warning        if (not python_warning and            sys.version_info.major == 3 and sys.version_info.minor < 7):             self.emit_msg(0,                          'Python 3.7 or later is required for correct results')            python_warning = True     def emit_msg(self, ln, msg, *, warning=True):        """Emit a message"""         if self.entry:            self.entry.emit_msg(ln, msg, warning=warning)            return         log_msg = f"{self.fname}:{ln} {msg}"         if warning:            self.config.log.warning(log_msg)        else:            self.config.log.info(log_msg)     def dump_section(self, start_new=True):        """        Dump section contents to arrays/hashes intended for that purpose.        """         if self.entry:            self.entry.dump_section(start_new)     # TODO: rename it to store_declaration after removal of kernel-doc.pl    def output_declaration(self, dtype, name, **args):        """        Store the entry into an entry array.         The actual output and output filters will be handled elsewhere.        """         item = KdocItem(name, self.fname, dtype,                        self.entry.declaration_start_line, **args)        item.warnings = self.entry.warnings         # Drop empty sections        # TODO: improve empty sections logic to emit warnings        sections = self.entry.sections        for section in ["Description", "Return"]:            if section in sections and not sections[section].rstrip():                del sections[section]        item.set_sections(sections, self.entry.sections_start_lines)        item.set_params(self.entry.parameterlist, self.entry.parameterdescs,                        self.entry.parametertypes,                        self.entry.parameterdesc_start_lines)        self.entries.append(item)         self.config.log.debug("Output: %s:%s = %s", dtype, name, pformat(args))     def emit_unused_warnings(self):        """        When the parser fails to produce a valid entry, it places some        warnings under `entry.warnings` that will be discarded when resetting        the state.         Ensure that those warnings are not lost.         .. note::               Because we are calling `config.warning()` here, those              warnings are not filtered by the `-W` parameters: they will all              be produced even when `-Wreturn`, `-Wshort-desc`, and/or              `-Wcontents-before-sections` are used.               Allowing those warnings to be filtered is complex, because it              would require storing them in a buffer and then filtering them              during the output step of the code, depending on the              selected symbols.        """        if self.entry and self.entry not in self.entries:            for log_msg in self.entry.warnings:                self.config.warning(log_msg)     def reset_state(self, ln):        """        Ancillary routine to create a new entry. It initializes all        variables used by the state machine.        """         self.emit_unused_warnings()         self.entry = KernelEntry(self.config, self.fname, ln)         # State flags        self.state = state.NORMAL     def push_parameter(self, ln, decl_type, param, dtype,                       org_arg, declaration_name):        """        Store parameters and their descriptions at self.entry.        """         if self.entry.anon_struct_union and dtype == "" and param == "}":            return  # Ignore the ending }; from anonymous struct/union         self.entry.anon_struct_union = False         param = KernRe(r'[\[\)].*').sub('', param, count=1)         #        # Look at various "anonymous type" cases.        #        if dtype == '':            if param.endswith("..."):                if len(param) > 3: # there is a name provided, use that                    param = param[:-3]                if not self.entry.parameterdescs.get(param):                    self.entry.parameterdescs[param] = "variable arguments"             elif (not param) or param == "void":                param = "void"                self.entry.parameterdescs[param] = "no arguments"             elif param in ["struct", "union"]:                # Handle unnamed (anonymous) union or struct                dtype = param                param = "{unnamed_" + param + "}"                self.entry.parameterdescs[param] = "anonymous\n"                self.entry.anon_struct_union = True         # Warn if parameter has no description        # (but ignore ones starting with # as these are not parameters        # but inline preprocessor statements)        if param not in self.entry.parameterdescs and not param.startswith("#"):            self.entry.parameterdescs[param] = self.undescribed             if "." not in param:                if decl_type == 'function':                    dname = f"{decl_type} parameter"                else:                    dname = f"{decl_type} member"                 self.emit_msg(ln,                              f"{dname} '{param}' not described in '{declaration_name}'")         # Strip spaces from param so that it is one continuous string on        # parameterlist. This fixes a problem where check_sections()        # cannot find a parameter like "addr[6 + 2]" because it actually        # appears as "addr[6", "+", "2]" on the parameter list.        # However, it's better to maintain the param string unchanged for        # output, so just weaken the string compare in check_sections()        # to ignore "[blah" in a parameter string.         self.entry.parameterlist.append(param)        org_arg = KernRe(r'\s\s+').sub(' ', org_arg)        self.entry.parametertypes[param] = org_arg      def create_parameter_list(self, ln, decl_type, args,                              splitter, declaration_name):        """        Creates a list of parameters, storing them at self.entry.        """         # temporarily replace all commas inside function pointer definition        arg_expr = KernRe(r'(\([^\),]+),')        while arg_expr.search(args):            args = arg_expr.sub(r"\1#", args)         for arg in args.split(splitter):            # Ignore argument attributes            arg = KernRe(r'\sPOS0?\s').sub(' ', arg)             # Replace '[at_least ' with '[static '.  This allows sphinx to parse            # array parameter declarations like 'char A[at_least 4]', where            # 'at_least' is #defined to 'static' by the kernel headers.            arg = arg.replace('[at_least ', '[static ')             # Strip leading/trailing spaces            arg = arg.strip()            arg = KernRe(r'\s+').sub(' ', arg, count=1)             if arg.startswith('#'):                # Treat preprocessor directive as a typeless variable just to fill                # corresponding data structures "correctly". Catch it later in                # output_* subs.                 # Treat preprocessor directive as a typeless variable                self.push_parameter(ln, decl_type, arg, "",                                    "", declaration_name)            #            # The pointer-to-function case.            #            elif KernRe(r'\(.+\)\s*\(').search(arg):                arg = arg.replace('#', ',')                r = KernRe(r'[^\(]+\(\*?\s*'  # Everything up to "(*"                           r'([\w\[\].]*)'    # Capture the name and possible [array]                           r'\s*\)')	      # Make sure the trailing ")" is there                if r.match(arg):                    param = r.group(1)                else:                    self.emit_msg(ln, f"Invalid param: {arg}")                    param = arg                dtype = arg.replace(param, '')                self.push_parameter(ln, decl_type, param, dtype, arg, declaration_name)            #            # The array-of-pointers case.  Dig the parameter name out from the middle            # of the declaration.            #            elif KernRe(r'\(.+\)\s*\[').search(arg):                r = KernRe(r'[^\(]+\(\s*\*\s*'		# Up to "(" and maybe "*"                           r'([\w.]*?)'			# The actual pointer name                           r'\s*(\[\s*\w+\s*\]\s*)*\)') # The [array portion]                if r.match(arg):                    param = r.group(1)                else:                    self.emit_msg(ln, f"Invalid param: {arg}")                    param = arg                dtype = arg.replace(param, '')                self.push_parameter(ln, decl_type, param, dtype, arg, declaration_name)            elif arg:                #                # Clean up extraneous spaces and split the string at commas; the first                # element of the resulting list will also include the type information.                #                arg = KernRe(r'\s*:\s*').sub(":", arg)                arg = KernRe(r'\s*\[').sub('[', arg)                args = KernRe(r'\s*,\s*').split(arg)                args[0] = re.sub(r'(\*+)\s*', r' \1', args[0])                #                # args[0] has a string of "type a".  If "a" includes an [array]                # declaration, we want to not be fooled by any white space inside

Lines 1–500 of 1679. Load more