1669 lines · 59.5 KB · Python
RawDownload
#!/usr/bin/env python3# SPDX-License-Identifier: GPL-2.0# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>.## pylint: disable=C0301,C0302,R0904,R0912,R0913,R0914,R0915,R0917,R1702 """kdoc_parser=========== Read a C language source or header FILE and extract embeddeddocumentation comments""" import sysimport refrom pprint import pformat from kdoc_re import NestedMatch, KernRefrom kdoc_item import KdocItem ## Regular expressions used to parse kernel-doc markups at KernelDoc class.## Let's declare them in lowercase outside any class to make easier to# convert from the python script.## As those are evaluated at the beginning, no need to cache them# # Allow whitespace at end of comment start.doc_start = KernRe(r'^/\*\*\s*$', cache=False) doc_end = KernRe(r'\*/', cache=False)doc_com = KernRe(r'\s*\*\s*', cache=False)doc_com_body = KernRe(r'\s*\* ?', cache=False)doc_decl = doc_com + KernRe(r'(\w+)', cache=False) # @params and a strictly limited set of supported section names# Specifically:#   Match @word:#         @...:#         @{section-name}:# while trying to not match literal block starts like "example::"#known_section_names = 'description|context|returns?|notes?|examples?'known_sections = KernRe(known_section_names, flags = re.I)doc_sect = doc_com + \    KernRe(r'\s*(\@[.\w]+|\@\.\.\.|' + known_section_names + r')\s*:([^:].*)?$',           flags=re.I, cache=False) doc_content = doc_com_body + KernRe(r'(.*)', cache=False)doc_inline_start = KernRe(r'^\s*/\*\*\s*$', cache=False)doc_inline_sect = KernRe(r'\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)', cache=False)doc_inline_end = KernRe(r'^\s*\*/\s*$', cache=False)doc_inline_oneline = KernRe(r'^\s*/\*\*\s*(@[\w\s]+):\s*(.*)\s*\*/\s*$', cache=False)attribute = KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)",               flags=re.I | re.S, cache=False) export_symbol = KernRe(r'^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*', cache=False)export_symbol_ns = KernRe(r'^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*"\S+"\)\s*', cache=False) type_param = KernRe(r"\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) ## Tests for the beginning of a kerneldoc block in its various forms.#doc_block = doc_com + KernRe(r'DOC:\s*(.*)?', cache=False)doc_begin_data = KernRe(r"^\s*\*?\s*(struct|union|enum|typedef)\b\s*(\w*)", cache = False)doc_begin_func = KernRe(str(doc_com) +			# initial " * '                        r"(?:\w+\s*\*\s*)?" + 		# type (not captured)                        r'(?:define\s+)?' + 		# possible "define" (not captured)                        r'(\w+)\s*(?:\(\w*\))?\s*' +	# name and optional "(...)"                        r'(?:[-:].*)?$',		# description (not captured)                        cache = False) ## A little helper to get rid of excess white space#multi_space = KernRe(r'\s\s+')def trim_whitespace(s):    return multi_space.sub(' ', s.strip()) class state:    """    State machine enums    """     # Parser states    NORMAL        = 0        # normal code    NAME          = 1        # looking for function name    DECLARATION   = 2        # We have seen a declaration which might not be done    BODY          = 3        # the body of the comment    SPECIAL_SECTION = 4      # doc section ending with a blank line    PROTO         = 5        # scanning prototype    DOCBLOCK      = 6        # documentation block    INLINE_NAME   = 7        # gathering doc outside main block    INLINE_TEXT   = 8	     # reading the body of inline docs     name = [        "NORMAL",        "NAME",        "DECLARATION",        "BODY",        "SPECIAL_SECTION",        "PROTO",        "DOCBLOCK",        "INLINE_NAME",        "INLINE_TEXT",    ]  SECTION_DEFAULT = "Description"  # default section class KernelEntry:     def __init__(self, config, ln):        self.config = config         self._contents = []        self.prototype = ""         self.warnings = []         self.parameterlist = []        self.parameterdescs = {}        self.parametertypes = {}        self.parameterdesc_start_lines = {}         self.section_start_lines = {}        self.sections = {}         self.anon_struct_union = False         self.leading_space = None         # State flags        self.brcount = 0        self.declaration_start_line = ln + 1     #    # Management of section contents    #    def add_text(self, text):        self._contents.append(text)     def contents(self):        return '\n'.join(self._contents) + '\n'     # TODO: rename to emit_message after removal of kernel-doc.pl    def emit_msg(self, log_msg, warning=True):        """Emit a message"""         if not warning:            self.config.log.info(log_msg)            return         # Delegate warning output to output logic, as this way it        # will report warnings/info only for symbols that are output         self.warnings.append(log_msg)        return     #    # Begin a new section.    #    def begin_section(self, line_no, title = SECTION_DEFAULT, dump = False):        if dump:            self.dump_section(start_new = True)        self.section = title        self.new_start_line = line_no     def dump_section(self, start_new=True):        """        Dumps section contents to arrays/hashes intended for that purpose.        """        #        # If we have accumulated no contents in the default ("description")        # section, don't bother.        #        if self.section == SECTION_DEFAULT and not self._contents:            return        name = self.section        contents = self.contents()         if type_param.match(name):            name = type_param.group(1)             self.parameterdescs[name] = contents            self.parameterdesc_start_lines[name] = self.new_start_line             self.new_start_line = 0         else:            if name in self.sections and self.sections[name] != "":                # Only warn on user-specified duplicate section names                if name != SECTION_DEFAULT:                    self.emit_msg(self.new_start_line,                                  f"duplicate section name '{name}'\n")                # Treat as a new paragraph - add a blank line                self.sections[name] += '\n' + contents            else:                self.sections[name] = contents                self.section_start_lines[name] = self.new_start_line                self.new_start_line = 0 #        self.config.log.debug("Section: %s : %s", name, pformat(vars(self)))         if start_new:            self.section = SECTION_DEFAULT            self._contents = []  class KernelDoc:    """    Read a C language source or header FILE and extract embedded    documentation comments.    """     # Section names     section_context = "Context"    section_return = "Return"     undescribed = "-- undescribed --"     def __init__(self, config, fname):        """Initialize internal variables"""         self.fname = fname        self.config = config         # Initial state for the state machines        self.state = state.NORMAL         # Store entry currently being processed        self.entry = None         # Place all potential outputs into an array        self.entries = []         #        # We need Python 3.7 for its "dicts remember the insertion        # order" guarantee        #        if sys.version_info.major == 3 and sys.version_info.minor < 7:            self.emit_msg(0,                          'Python 3.7 or later is required for correct results')     def emit_msg(self, ln, msg, warning=True):        """Emit a message"""         log_msg = f"{self.fname}:{ln} {msg}"         if self.entry:            self.entry.emit_msg(log_msg, warning)            return         if warning:            self.config.log.warning(log_msg)        else:            self.config.log.info(log_msg)     def dump_section(self, start_new=True):        """        Dumps section contents to arrays/hashes intended for that purpose.        """         if self.entry:            self.entry.dump_section(start_new)     # TODO: rename it to store_declaration after removal of kernel-doc.pl    def output_declaration(self, dtype, name, **args):        """        Stores the entry into an entry array.         The actual output and output filters will be handled elsewhere        """         item = KdocItem(name, dtype, self.entry.declaration_start_line, **args)        item.warnings = self.entry.warnings         # Drop empty sections        # TODO: improve empty sections logic to emit warnings        sections = self.entry.sections        for section in ["Description", "Return"]:            if section in sections and not sections[section].rstrip():                del sections[section]        item.set_sections(sections, self.entry.section_start_lines)        item.set_params(self.entry.parameterlist, self.entry.parameterdescs,                        self.entry.parametertypes,                        self.entry.parameterdesc_start_lines)        self.entries.append(item)         self.config.log.debug("Output: %s:%s = %s", dtype, name, pformat(args))     def reset_state(self, ln):        """        Ancillary routine to create a new entry. It initializes all        variables used by the state machine.        """         self.entry = KernelEntry(self.config, ln)         # State flags        self.state = state.NORMAL     def push_parameter(self, ln, decl_type, param, dtype,                       org_arg, declaration_name):        """        Store parameters and their descriptions at self.entry.        """         if self.entry.anon_struct_union and dtype == "" and param == "}":            return  # Ignore the ending }; from anonymous struct/union         self.entry.anon_struct_union = False         param = KernRe(r'[\[\)].*').sub('', param, count=1)         if dtype == "" and param.endswith("..."):            if KernRe(r'\w\.\.\.$').search(param):                # For named variable parameters of the form `x...`,                # remove the dots                param = param[:-3]            else:                # Handles unnamed variable parameters                param = "..."             if param not in self.entry.parameterdescs or \                not self.entry.parameterdescs[param]:                 self.entry.parameterdescs[param] = "variable arguments"         elif dtype == "" and (not param or param == "void"):            param = "void"            self.entry.parameterdescs[param] = "no arguments"         elif dtype == "" and param in ["struct", "union"]:            # Handle unnamed (anonymous) union or struct            dtype = param            param = "{unnamed_" + param + "}"            self.entry.parameterdescs[param] = "anonymous\n"            self.entry.anon_struct_union = True         # Handle cache group enforcing variables: they do not need        # to be described in header files        elif "__cacheline_group" in param:            # Ignore __cacheline_group_begin and __cacheline_group_end            return         # Warn if parameter has no description        # (but ignore ones starting with # as these are not parameters        # but inline preprocessor statements)        if param not in self.entry.parameterdescs and not param.startswith("#"):            self.entry.parameterdescs[param] = self.undescribed             if "." not in param:                if decl_type == 'function':                    dname = f"{decl_type} parameter"                else:                    dname = f"{decl_type} member"                 self.emit_msg(ln,                              f"{dname} '{param}' not described in '{declaration_name}'")         # Strip spaces from param so that it is one continuous string on        # parameterlist. This fixes a problem where check_sections()        # cannot find a parameter like "addr[6 + 2]" because it actually        # appears as "addr[6", "+", "2]" on the parameter list.        # However, it's better to maintain the param string unchanged for        # output, so just weaken the string compare in check_sections()        # to ignore "[blah" in a parameter string.         self.entry.parameterlist.append(param)        org_arg = KernRe(r'\s\s+').sub(' ', org_arg)        self.entry.parametertypes[param] = org_arg      def create_parameter_list(self, ln, decl_type, args,                              splitter, declaration_name):        """        Creates a list of parameters, storing them at self.entry.        """         # temporarily replace all commas inside function pointer definition        arg_expr = KernRe(r'(\([^\),]+),')        while arg_expr.search(args):            args = arg_expr.sub(r"\1#", args)         for arg in args.split(splitter):            # Strip comments            arg = KernRe(r'\/\*.*\*\/').sub('', arg)             # Ignore argument attributes            arg = KernRe(r'\sPOS0?\s').sub(' ', arg)             # Strip leading/trailing spaces            arg = arg.strip()            arg = KernRe(r'\s+').sub(' ', arg, count=1)             if arg.startswith('#'):                # Treat preprocessor directive as a typeless variable just to fill                # corresponding data structures "correctly". Catch it later in                # output_* subs.                 # Treat preprocessor directive as a typeless variable                self.push_parameter(ln, decl_type, arg, "",                                    "", declaration_name)             elif KernRe(r'\(.+\)\s*\(').search(arg):                # Pointer-to-function                 arg = arg.replace('#', ',')                 r = KernRe(r'[^\(]+\(\*?\s*([\w\[\]\.]*)\s*\)')                if r.match(arg):                    param = r.group(1)                else:                    self.emit_msg(ln, f"Invalid param: {arg}")                    param = arg                 dtype = KernRe(r'([^\(]+\(\*?)\s*' + re.escape(param)).sub(r'\1', arg)                self.push_parameter(ln, decl_type, param, dtype,                                    arg, declaration_name)             elif KernRe(r'\(.+\)\s*\[').search(arg):                # Array-of-pointers                 arg = arg.replace('#', ',')                r = KernRe(r'[^\(]+\(\s*\*\s*([\w\[\]\.]*?)\s*(\s*\[\s*[\w]+\s*\]\s*)*\)')                if r.match(arg):                    param = r.group(1)                else:                    self.emit_msg(ln, f"Invalid param: {arg}")                    param = arg                 dtype = KernRe(r'([^\(]+\(\*?)\s*' + re.escape(param)).sub(r'\1', arg)                 self.push_parameter(ln, decl_type, param, dtype,                                    arg, declaration_name)             elif arg:                arg = KernRe(r'\s*:\s*').sub(":", arg)                arg = KernRe(r'\s*\[').sub('[', arg)                 args = KernRe(r'\s*,\s*').split(arg)                if args[0] and '*' in args[0]:                    args[0] = re.sub(r'(\*+)\s*', r' \1', args[0])                 first_arg = []                r = KernRe(r'^(.*\s+)(.*?\[.*\].*)$')                if args[0] and r.match(args[0]):                    args.pop(0)                    first_arg.extend(r.group(1))                    first_arg.append(r.group(2))                else:                    first_arg = KernRe(r'\s+').split(args.pop(0))                 args.insert(0, first_arg.pop())                dtype = ' '.join(first_arg)                 for param in args:                    if KernRe(r'^(\*+)\s*(.*)').match(param):                        r = KernRe(r'^(\*+)\s*(.*)')                        if not r.match(param):                            self.emit_msg(ln, f"Invalid param: {param}")                            continue                         param = r.group(1)                         self.push_parameter(ln, decl_type, r.group(2),                                            f"{dtype} {r.group(1)}",                                            arg, declaration_name)                     elif KernRe(r'(.*?):(\w+)').search(param):                        r = KernRe(r'(.*?):(\w+)')                        if not r.match(param):                            self.emit_msg(ln, f"Invalid param: {param}")                            continue                         if dtype != "":  # Skip unnamed bit-fields                            self.push_parameter(ln, decl_type, r.group(1),                                                f"{dtype}:{r.group(2)}",                                                arg, declaration_name)                    else:                        self.push_parameter(ln, decl_type, param, dtype,                                            arg, declaration_name)     def check_sections(self, ln, decl_name, decl_type):        """        Check for errors inside sections, emitting warnings if not found        parameters are described.        """        for section in self.entry.sections:            if section not in self.entry.parameterlist and \               not known_sections.search(section):                if decl_type == 'function':                    dname = f"{decl_type} parameter"                else:

Lines 1–500 of 1669.