1745 lines · 63.7 KB · Python
RawDownload
#!/usr/bin/env python3# SPDX-License-Identifier: GPL-2.0# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>.## pylint: disable=C0301,C0302,R0904,R0912,R0913,R0914,R0915,R0917,R1702 """kdoc_parser=========== Read a C language source or header FILE and extract embeddeddocumentation comments""" import refrom pprint import pformat from kdoc_re import NestedMatch, KernRe  ## Regular expressions used to parse kernel-doc markups at KernelDoc class.## Let's declare them in lowercase outside any class to make easier to# convert from the python script.## As those are evaluated at the beginning, no need to cache them# # Allow whitespace at end of comment start.doc_start = KernRe(r'^/\*\*\s*$', cache=False) doc_end = KernRe(r'\*/', cache=False)doc_com = KernRe(r'\s*\*\s*', cache=False)doc_com_body = KernRe(r'\s*\* ?', cache=False)doc_decl = doc_com + KernRe(r'(\w+)', cache=False) # @params and a strictly limited set of supported section names# Specifically:#   Match @word:#         @...:#         @{section-name}:# while trying to not match literal block starts like "example::"#doc_sect = doc_com + \            KernRe(r'\s*(\@[.\w]+|\@\.\.\.|description|context|returns?|notes?|examples?)\s*:([^:].*)?$',                flags=re.I, cache=False) doc_content = doc_com_body + KernRe(r'(.*)', cache=False)doc_block = doc_com + KernRe(r'DOC:\s*(.*)?', cache=False)doc_inline_start = KernRe(r'^\s*/\*\*\s*$', cache=False)doc_inline_sect = KernRe(r'\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)', cache=False)doc_inline_end = KernRe(r'^\s*\*/\s*$', cache=False)doc_inline_oneline = KernRe(r'^\s*/\*\*\s*(@[\w\s]+):\s*(.*)\s*\*/\s*$', cache=False)attribute = KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)",               flags=re.I | re.S, cache=False) export_symbol = KernRe(r'^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*', cache=False)export_symbol_ns = KernRe(r'^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*"\S+"\)\s*', cache=False) type_param = KernRe(r"\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) class state:    """    State machine enums    """     # Parser states    NORMAL        = 0        # normal code    NAME          = 1        # looking for function name    BODY_MAYBE    = 2        # body - or maybe more description    BODY          = 3        # the body of the comment    BODY_WITH_BLANK_LINE = 4 # the body which has a blank line    PROTO         = 5        # scanning prototype    DOCBLOCK      = 6        # documentation block    INLINE        = 7        # gathering doc outside main block     name = [        "NORMAL",        "NAME",        "BODY_MAYBE",        "BODY",        "BODY_WITH_BLANK_LINE",        "PROTO",        "DOCBLOCK",        "INLINE",    ]     # Inline documentation state    INLINE_NA     = 0 # not applicable ($state != INLINE)    INLINE_NAME   = 1 # looking for member name (@foo:)    INLINE_TEXT   = 2 # looking for member documentation    INLINE_END    = 3 # done    INLINE_ERROR  = 4 # error - Comment without header was found.                      # Spit a warning as it's not                      # proper kernel-doc and ignore the rest.     inline_name = [        "",        "_NAME",        "_TEXT",        "_END",        "_ERROR",    ] SECTION_DEFAULT = "Description"  # default section class KernelEntry:     def __init__(self, config, ln):        self.config = config         self.contents = ""        self.function = ""        self.sectcheck = ""        self.struct_actual = ""        self.prototype = ""         self.warnings = []         self.parameterlist = []        self.parameterdescs = {}        self.parametertypes = {}        self.parameterdesc_start_lines = {}         self.section_start_lines = {}        self.sectionlist = []        self.sections = {}         self.anon_struct_union = False         self.leading_space = None         # State flags        self.brcount = 0         self.in_doc_sect = False        self.declaration_start_line = ln + 1     # TODO: rename to emit_message after removal of kernel-doc.pl    def emit_msg(self, log_msg, warning=True):        """Emit a message"""         if not warning:            self.config.log.info(log_msg)            return         # Delegate warning output to output logic, as this way it        # will report warnings/info only for symbols that are output         self.warnings.append(log_msg)        return     def dump_section(self, start_new=True):        """        Dumps section contents to arrays/hashes intended for that purpose.        """         name = self.section        contents = self.contents         if type_param.match(name):            name = type_param.group(1)             self.parameterdescs[name] = contents            self.parameterdesc_start_lines[name] = self.new_start_line             self.sectcheck += name + " "            self.new_start_line = 0         elif name == "@...":            name = "..."            self.parameterdescs[name] = contents            self.sectcheck += name + " "            self.parameterdesc_start_lines[name] = self.new_start_line            self.new_start_line = 0         else:            if name in self.sections and self.sections[name] != "":                # Only warn on user-specified duplicate section names                if name != SECTION_DEFAULT:                    self.emit_msg(self.new_start_line,                                  f"duplicate section name '{name}'\n")                self.sections[name] += contents            else:                self.sections[name] = contents                self.sectionlist.append(name)                self.section_start_lines[name] = self.new_start_line                self.new_start_line = 0 #        self.config.log.debug("Section: %s : %s", name, pformat(vars(self)))         if start_new:            self.section = SECTION_DEFAULT            self.contents = ""  class KernelDoc:    """    Read a C language source or header FILE and extract embedded    documentation comments.    """     # Section names     section_intro = "Introduction"    section_context = "Context"    section_return = "Return"     undescribed = "-- undescribed --"     def __init__(self, config, fname):        """Initialize internal variables"""         self.fname = fname        self.config = config         # Initial state for the state machines        self.state = state.NORMAL        self.inline_doc_state = state.INLINE_NA         # Store entry currently being processed        self.entry = None         # Place all potential outputs into an array        self.entries = []     def emit_msg(self, ln, msg, warning=True):        """Emit a message"""         log_msg = f"{self.fname}:{ln} {msg}"         if self.entry:            self.entry.emit_msg(log_msg, warning)            return         if warning:            self.config.log.warning(log_msg)        else:            self.config.log.info(log_msg)     def dump_section(self, start_new=True):        """        Dumps section contents to arrays/hashes intended for that purpose.        """         if self.entry:            self.entry.dump_section(start_new)     # TODO: rename it to store_declaration after removal of kernel-doc.pl    def output_declaration(self, dtype, name, **args):        """        Stores the entry into an entry array.         The actual output and output filters will be handled elsewhere        """         # The implementation here is different than the original kernel-doc:        # instead of checking for output filters or actually output anything,        # it just stores the declaration content at self.entries, as the        # output will happen on a separate class.        #        # For now, we're keeping the same name of the function just to make        # easier to compare the source code of both scripts         args["declaration_start_line"] = self.entry.declaration_start_line        args["type"] = dtype        args["warnings"] = self.entry.warnings         # TODO: use colletions.OrderedDict to remove sectionlist         sections = args.get('sections', {})        sectionlist = args.get('sectionlist', [])         # Drop empty sections        # TODO: improve empty sections logic to emit warnings        for section in ["Description", "Return"]:            if section in sectionlist:                if not sections[section].rstrip():                    del sections[section]                    sectionlist.remove(section)         self.entries.append((name, args))         self.config.log.debug("Output: %s:%s = %s", dtype, name, pformat(args))     def reset_state(self, ln):        """        Ancillary routine to create a new entry. It initializes all        variables used by the state machine.        """         self.entry = KernelEntry(self.config, ln)         # State flags        self.state = state.NORMAL        self.inline_doc_state = state.INLINE_NA     def push_parameter(self, ln, decl_type, param, dtype,                       org_arg, declaration_name):        """        Store parameters and their descriptions at self.entry.        """         if self.entry.anon_struct_union and dtype == "" and param == "}":            return  # Ignore the ending }; from anonymous struct/union         self.entry.anon_struct_union = False         param = KernRe(r'[\[\)].*').sub('', param, count=1)         if dtype == "" and param.endswith("..."):            if KernRe(r'\w\.\.\.$').search(param):                # For named variable parameters of the form `x...`,                # remove the dots                param = param[:-3]            else:                # Handles unnamed variable parameters                param = "..."             if param not in self.entry.parameterdescs or \                not self.entry.parameterdescs[param]:                 self.entry.parameterdescs[param] = "variable arguments"         elif dtype == "" and (not param or param == "void"):            param = "void"            self.entry.parameterdescs[param] = "no arguments"         elif dtype == "" and param in ["struct", "union"]:            # Handle unnamed (anonymous) union or struct            dtype = param            param = "{unnamed_" + param + "}"            self.entry.parameterdescs[param] = "anonymous\n"            self.entry.anon_struct_union = True         # Handle cache group enforcing variables: they do not need        # to be described in header files        elif "__cacheline_group" in param:            # Ignore __cacheline_group_begin and __cacheline_group_end            return         # Warn if parameter has no description        # (but ignore ones starting with # as these are not parameters        # but inline preprocessor statements)        if param not in self.entry.parameterdescs and not param.startswith("#"):            self.entry.parameterdescs[param] = self.undescribed             if "." not in param:                if decl_type == 'function':                    dname = f"{decl_type} parameter"                else:                    dname = f"{decl_type} member"                 self.emit_msg(ln,                              f"{dname} '{param}' not described in '{declaration_name}'")         # Strip spaces from param so that it is one continuous string on        # parameterlist. This fixes a problem where check_sections()        # cannot find a parameter like "addr[6 + 2]" because it actually        # appears as "addr[6", "+", "2]" on the parameter list.        # However, it's better to maintain the param string unchanged for        # output, so just weaken the string compare in check_sections()        # to ignore "[blah" in a parameter string.         self.entry.parameterlist.append(param)        org_arg = KernRe(r'\s\s+').sub(' ', org_arg)        self.entry.parametertypes[param] = org_arg     def save_struct_actual(self, actual):        """        Strip all spaces from the actual param so that it looks like        one string item.        """         actual = KernRe(r'\s*').sub("", actual, count=1)         self.entry.struct_actual += actual + " "     def create_parameter_list(self, ln, decl_type, args,                              splitter, declaration_name):        """        Creates a list of parameters, storing them at self.entry.        """         # temporarily replace all commas inside function pointer definition        arg_expr = KernRe(r'(\([^\),]+),')        while arg_expr.search(args):            args = arg_expr.sub(r"\1#", args)         for arg in args.split(splitter):            # Strip comments            arg = KernRe(r'\/\*.*\*\/').sub('', arg)             # Ignore argument attributes            arg = KernRe(r'\sPOS0?\s').sub(' ', arg)             # Strip leading/trailing spaces            arg = arg.strip()            arg = KernRe(r'\s+').sub(' ', arg, count=1)             if arg.startswith('#'):                # Treat preprocessor directive as a typeless variable just to fill                # corresponding data structures "correctly". Catch it later in                # output_* subs.                 # Treat preprocessor directive as a typeless variable                self.push_parameter(ln, decl_type, arg, "",                                    "", declaration_name)             elif KernRe(r'\(.+\)\s*\(').search(arg):                # Pointer-to-function                 arg = arg.replace('#', ',')                 r = KernRe(r'[^\(]+\(\*?\s*([\w\[\]\.]*)\s*\)')                if r.match(arg):                    param = r.group(1)                else:                    self.emit_msg(ln, f"Invalid param: {arg}")                    param = arg                 dtype = KernRe(r'([^\(]+\(\*?)\s*' + re.escape(param)).sub(r'\1', arg)                self.save_struct_actual(param)                self.push_parameter(ln, decl_type, param, dtype,                                    arg, declaration_name)             elif KernRe(r'\(.+\)\s*\[').search(arg):                # Array-of-pointers                 arg = arg.replace('#', ',')                r = KernRe(r'[^\(]+\(\s*\*\s*([\w\[\]\.]*?)\s*(\s*\[\s*[\w]+\s*\]\s*)*\)')                if r.match(arg):                    param = r.group(1)                else:                    self.emit_msg(ln, f"Invalid param: {arg}")                    param = arg                 dtype = KernRe(r'([^\(]+\(\*?)\s*' + re.escape(param)).sub(r'\1', arg)                 self.save_struct_actual(param)                self.push_parameter(ln, decl_type, param, dtype,                                    arg, declaration_name)             elif arg:                arg = KernRe(r'\s*:\s*').sub(":", arg)                arg = KernRe(r'\s*\[').sub('[', arg)                 args = KernRe(r'\s*,\s*').split(arg)                if args[0] and '*' in args[0]:                    args[0] = re.sub(r'(\*+)\s*', r' \1', args[0])                 first_arg = []                r = KernRe(r'^(.*\s+)(.*?\[.*\].*)$')                if args[0] and r.match(args[0]):                    args.pop(0)                    first_arg.extend(r.group(1))                    first_arg.append(r.group(2))                else:                    first_arg = KernRe(r'\s+').split(args.pop(0))                 args.insert(0, first_arg.pop())                dtype = ' '.join(first_arg)                 for param in args:                    if KernRe(r'^(\*+)\s*(.*)').match(param):                        r = KernRe(r'^(\*+)\s*(.*)')                        if not r.match(param):                            self.emit_msg(ln, f"Invalid param: {param}")                            continue                         param = r.group(1)                         self.save_struct_actual(r.group(2))                        self.push_parameter(ln, decl_type, r.group(2),                                            f"{dtype} {r.group(1)}",                                            arg, declaration_name)                     elif KernRe(r'(.*?):(\w+)').search(param):                        r = KernRe(r'(.*?):(\w+)')                        if not r.match(param):                            self.emit_msg(ln, f"Invalid param: {param}")                            continue                         if dtype != "":  # Skip unnamed bit-fields                            self.save_struct_actual(r.group(1))                            self.push_parameter(ln, decl_type, r.group(1),                                                f"{dtype}:{r.group(2)}",                                                arg, declaration_name)                    else:                        self.save_struct_actual(param)                        self.push_parameter(ln, decl_type, param, dtype,                                            arg, declaration_name)     def check_sections(self, ln, decl_name, decl_type, sectcheck, prmscheck):        """        Check for errors inside sections, emitting warnings if not found        parameters are described.        """ 

Lines 1–500 of 1745.