#!/usr/bin/env python3# SPDX-License-Identifier: GPL-2.0# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>.## pylint: disable=C0301,C0302,R0904,R0912,R0913,R0914,R0915,R0917,R1702 """kdoc_parser=========== Read a C language source or header FILE and extract embeddeddocumentation comments""" import sysimport refrom pprint import pformat from kdoc_re import NestedMatch, KernRefrom kdoc_item import KdocItem ## Regular expressions used to parse kernel-doc markups at KernelDoc class.## Let's declare them in lowercase outside any class to make easier to# convert from the python script.## As those are evaluated at the beginning, no need to cache them# # Allow whitespace at end of comment start.doc_start = KernRe(r'^/\*\*\s*$', cache=False) doc_end = KernRe(r'\*/', cache=False)doc_com = KernRe(r'\s*\*\s*', cache=False)doc_com_body = KernRe(r'\s*\* ?', cache=False)doc_decl = doc_com + KernRe(r'(\w+)', cache=False) # @params and a strictly limited set of supported section names# Specifically:# Match @word:# @...:# @{section-name}:# while trying to not match literal block starts like "example::"#known_section_names = 'description|context|returns?|notes?|examples?'known_sections = KernRe(known_section_names, flags = re.I)doc_sect = doc_com + \ KernRe(r'\s*(@[.\w]+|@\.\.\.|' + known_section_names + r')\s*:([^:].*)?$', flags=re.I, cache=False) doc_content = doc_com_body + KernRe(r'(.*)', cache=False)doc_inline_start = KernRe(r'^\s*/\*\*\s*$', cache=False)doc_inline_sect = KernRe(r'\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)', cache=False)doc_inline_end = KernRe(r'^\s*\*/\s*$', cache=False)doc_inline_oneline = KernRe(r'^\s*/\*\*\s*(@[\w\s]+):\s*(.*)\s*\*/\s*$', cache=False) export_symbol = KernRe(r'^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*', cache=False)export_symbol_ns = KernRe(r'^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*"\S+"\)\s*', cache=False) type_param = KernRe(r"@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) ## Tests for the beginning of a kerneldoc block in its various forms.#doc_block = doc_com + KernRe(r'DOC:\s*(.*)?', cache=False)doc_begin_data = KernRe(r"^\s*\*?\s*(struct|union|enum|typedef)\b\s*(\w*)", cache = False)doc_begin_func = KernRe(str(doc_com) + # initial " * ' r"(?:\w+\s*\*\s*)?" + # type (not captured) r'(?:define\s+)?' + # possible "define" (not captured) r'(\w+)\s*(?:\(\w*\))?\s*' + # name and optional "(...)" r'(?:[-:].*)?$', # description (not captured) cache = False) ## Here begins a long set of transformations to turn structure member prefixes# and macro invocations into something we can parse and generate kdoc for.#struct_args_pattern = r'([^,)]+)' struct_xforms = [ # Strip attributes (KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", flags=re.I | re.S, cache=False), ' '), (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__packed\s*', re.S), ' '), (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), (KernRe(r'\s*____cacheline_aligned', re.S), ' '), (KernRe(r'\s*__cacheline_group_(begin|end)\([^\)]+\);'), ''), # # Unwrap struct_group macros based on this definition: # __struct_group(TAG, NAME, ATTRS, MEMBERS...) # which has variants like: struct_group(NAME, MEMBERS...) # Only MEMBERS arguments require documentation. # # Parsing them happens on two steps: # # 1. drop struct group arguments that aren't at MEMBERS, # storing them as STRUCT_GROUP(MEMBERS) # # 2. remove STRUCT_GROUP() ancillary macro. # # The original logic used to remove STRUCT_GROUP() using an # advanced regex: # # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; # # with two patterns that are incompatible with # Python re module, as it has: # # - a recursive pattern: (?1) # - an atomic grouping: (?>...) # # I tried a simpler version: but it didn't work either: # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; # # As it doesn't properly match the end parenthesis on some cases. # # So, a better solution was crafted: there's now a NestedMatch # class that ensures that delimiters after a search are properly # matched. So, the implementation to drop STRUCT_GROUP() will be # handled in separate. # (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), # # Replace macros # # TODO: use NestedMatch for FOO($1, $2, ...) matches # # it is better to also move those to the NestedMatch logic, # to ensure that parenthesis will be properly matched. # (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), (KernRe(r'DECLARE_BITMAP\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), (KernRe(r'DECLARE_HASHTABLE\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'unsigned long \1[1 << ((\2) - 1)]'), (KernRe(r'DECLARE_KFIFO\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'\1 \2[]'), (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + struct_args_pattern + r'\)', re.S), r'dma_addr_t \1'), (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + struct_args_pattern + r'\)', re.S), r'__u32 \1'),]## Regexes here are guaranteed to have the end limiter matching# the start delimiter. Yet, right now, only one replace group# is allowed.#struct_nested_prefixes = [ (re.compile(r'\bSTRUCT_GROUP\('), r'\1'),] ## Transforms for function prototypes#function_xforms = [ (KernRe(r"^static +"), ""), (KernRe(r"^extern +"), ""), (KernRe(r"^asmlinkage +"), ""), (KernRe(r"^inline +"), ""), (KernRe(r"^__inline__ +"), ""), (KernRe(r"^__inline +"), ""), (KernRe(r"^__always_inline +"), ""), (KernRe(r"^noinline +"), ""), (KernRe(r"^__FORTIFY_INLINE +"), ""), (KernRe(r"__init +"), ""), (KernRe(r"__init_or_module +"), ""), (KernRe(r"__deprecated +"), ""), (KernRe(r"__flatten +"), ""), (KernRe(r"__meminit +"), ""), (KernRe(r"__must_check +"), ""), (KernRe(r"__weak +"), ""), (KernRe(r"__sched +"), ""), (KernRe(r"_noprof"), ""), (KernRe(r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +"), ""), (KernRe(r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +"), ""), (KernRe(r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +"), ""), (KernRe(r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)"), r"\1, \2"), (KernRe(r"__attribute_const__ +"), ""), (KernRe(r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+"), ""),] ## Apply a set of transforms to a block of text.#def apply_transforms(xforms, text): for search, subst in xforms: text = search.sub(subst, text) return text ## A little helper to get rid of excess white space#multi_space = KernRe(r'\s\s+')def trim_whitespace(s): return multi_space.sub(' ', s.strip()) ## Remove struct/enum members that have been marked "private".#def trim_private_members(text): # # First look for a "public:" block that ends a private region, then # handle the "private until the end" case. # text = KernRe(r'/\*\s*private:.*?/\*\s*public:.*?\*/', flags=re.S).sub('', text) text = KernRe(r'/\*\s*private:.*', flags=re.S).sub('', text) # # We needed the comments to do the above, but now we can take them out. # return KernRe(r'\s*/\*.*?\*/\s*', flags=re.S).sub('', text).strip() class state: """ State machine enums """ # Parser states NORMAL = 0 # normal code NAME = 1 # looking for function name DECLARATION = 2 # We have seen a declaration which might not be done BODY = 3 # the body of the comment SPECIAL_SECTION = 4 # doc section ending with a blank line PROTO = 5 # scanning prototype DOCBLOCK = 6 # documentation block INLINE_NAME = 7 # gathering doc outside main block INLINE_TEXT = 8 # reading the body of inline docs name = [ "NORMAL", "NAME", "DECLARATION", "BODY", "SPECIAL_SECTION", "PROTO", "DOCBLOCK", "INLINE_NAME", "INLINE_TEXT", ] SECTION_DEFAULT = "Description" # default section class KernelEntry: def __init__(self, config, ln): self.config = config self._contents = [] self.prototype = "" self.warnings = [] self.parameterlist = [] self.parameterdescs = {} self.parametertypes = {} self.parameterdesc_start_lines = {} self.section_start_lines = {} self.sections = {} self.anon_struct_union = False self.leading_space = None # State flags self.brcount = 0 self.declaration_start_line = ln + 1 # # Management of section contents # def add_text(self, text): self._contents.append(text) def contents(self): return '\n'.join(self._contents) + '\n' # TODO: rename to emit_message after removal of kernel-doc.pl def emit_msg(self, log_msg, warning=True): """Emit a message""" if not warning: self.config.log.info(log_msg) return # Delegate warning output to output logic, as this way it # will report warnings/info only for symbols that are output self.warnings.append(log_msg) return # # Begin a new section. # def begin_section(self, line_no, title = SECTION_DEFAULT, dump = False): if dump: self.dump_section(start_new = True) self.section = title self.new_start_line = line_no def dump_section(self, start_new=True): """ Dumps section contents to arrays/hashes intended for that purpose. """ # # If we have accumulated no contents in the default ("description") # section, don't bother. # if self.section == SECTION_DEFAULT and not self._contents: return name = self.section contents = self.contents() if type_param.match(name): name = type_param.group(1) self.parameterdescs[name] = contents self.parameterdesc_start_lines[name] = self.new_start_line self.new_start_line = 0 else: if name in self.sections and self.sections[name] != "": # Only warn on user-specified duplicate section names if name != SECTION_DEFAULT: self.emit_msg(self.new_start_line, f"duplicate section name '{name}'\n") # Treat as a new paragraph - add a blank line self.sections[name] += '\n' + contents else: self.sections[name] = contents self.section_start_lines[name] = self.new_start_line self.new_start_line = 0 # self.config.log.debug("Section: %s : %s", name, pformat(vars(self))) if start_new: self.section = SECTION_DEFAULT self._contents = [] class KernelDoc: """ Read a C language source or header FILE and extract embedded documentation comments. """ # Section names section_context = "Context" section_return = "Return" undescribed = "-- undescribed --" def __init__(self, config, fname): """Initialize internal variables""" self.fname = fname self.config = config # Initial state for the state machines self.state = state.NORMAL # Store entry currently being processed self.entry = None # Place all potential outputs into an array self.entries = [] # # We need Python 3.7 for its "dicts remember the insertion # order" guarantee # if sys.version_info.major == 3 and sys.version_info.minor < 7: self.emit_msg(0, 'Python 3.7 or later is required for correct results') def emit_msg(self, ln, msg, warning=True): """Emit a message""" log_msg = f"{self.fname}:{ln} {msg}" if self.entry: self.entry.emit_msg(log_msg, warning) return if warning: self.config.log.warning(log_msg) else: self.config.log.info(log_msg) def dump_section(self, start_new=True): """ Dumps section contents to arrays/hashes intended for that purpose. """ if self.entry: self.entry.dump_section(start_new) # TODO: rename it to store_declaration after removal of kernel-doc.pl def output_declaration(self, dtype, name, **args): """ Stores the entry into an entry array. The actual output and output filters will be handled elsewhere """ item = KdocItem(name, dtype, self.entry.declaration_start_line, **args) item.warnings = self.entry.warnings # Drop empty sections # TODO: improve empty sections logic to emit warnings sections = self.entry.sections for section in ["Description", "Return"]: if section in sections and not sections[section].rstrip(): del sections[section] item.set_sections(sections, self.entry.section_start_lines) item.set_params(self.entry.parameterlist, self.entry.parameterdescs, self.entry.parametertypes, self.entry.parameterdesc_start_lines) self.entries.append(item) self.config.log.debug("Output: %s:%s = %s", dtype, name, pformat(args)) def reset_state(self, ln): """ Ancillary routine to create a new entry. It initializes all variables used by the state machine. """ self.entry = KernelEntry(self.config, ln) # State flags self.state = state.NORMAL def push_parameter(self, ln, decl_type, param, dtype, org_arg, declaration_name): """ Store parameters and their descriptions at self.entry. """ if self.entry.anon_struct_union and dtype == "" and param == "}": return # Ignore the ending }; from anonymous struct/union self.entry.anon_struct_union = False param = KernRe(r'[\[\)].*').sub('', param, count=1) # # Look at various "anonymous type" cases. # if dtype == '': if param.endswith("..."): if len(param) > 3: # there is a name provided, use that param = param[:-3] if not self.entry.parameterdescs.get(param): self.entry.parameterdescs[param] = "variable arguments" elif (not param) or param == "void": param = "void" self.entry.parameterdescs[param] = "no arguments" elif param in ["struct", "union"]: # Handle unnamed (anonymous) union or struct dtype = param param = "{unnamed_" + param + "}" self.entry.parameterdescs[param] = "anonymous\n" self.entry.anon_struct_union = True # Warn if parameter has no description # (but ignore ones starting with # as these are not parameters # but inline preprocessor statements) if param not in self.entry.parameterdescs and not param.startswith("#"): self.entry.parameterdescs[param] = self.undescribed if "." not in param: if decl_type == 'function': dname = f"{decl_type} parameter" else: dname = f"{decl_type} member" self.emit_msg(ln, f"{dname} '{param}' not described in '{declaration_name}'") # Strip spaces from param so that it is one continuous string on # parameterlist. This fixes a problem where check_sections() # cannot find a parameter like "addr[6 + 2]" because it actually # appears as "addr[6", "+", "2]" on the parameter list.Lines 1–500 of 1649. Load more
#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 # Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. # # pylint: disable=C0301,C0302,R0904,R0912,R0913,R0914,R0915,R0917,R1702 """ kdoc_parser =========== Read a C language source or header FILE and extract embedded documentation comments """ import sys import re from pprint import pformat from kdoc_re import NestedMatch, KernRe from kdoc_item import KdocItem # # Regular expressions used to parse kernel-doc markups at KernelDoc class. # # Let's declare them in lowercase outside any class to make easier to # convert from the python script. # # As those are evaluated at the beginning, no need to cache them # # Allow whitespace at end of comment start. doc_start = KernRe(r'^/\*\*\s*$', cache=False) doc_end = KernRe(r'\*/', cache=False) doc_com = KernRe(r'\s*\*\s*', cache=False) doc_com_body = KernRe(r'\s*\* ?', cache=False) doc_decl = doc_com + KernRe(r'(\w+)', cache=False) # @params and a strictly limited set of supported section names # Specifically: # Match @word: # @...: # @{section-name}: # while trying to not match literal block starts like "example::" # known_section_names = 'description|context|returns?|notes?|examples?' known_sections = KernRe(known_section_names, flags = re.I) doc_sect = doc_com + \ KernRe(r'\s*(@[.\w]+|@\.\.\.|' + known_section_names + r')\s*:([^:].*)?$', flags=re.I, cache=False) doc_content = doc_com_body + KernRe(r'(.*)', cache=False) doc_inline_start = KernRe(r'^\s*/\*\*\s*$', cache=False) doc_inline_sect = KernRe(r'\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)', cache=False) doc_inline_end = KernRe(r'^\s*\*/\s*$', cache=False) doc_inline_oneline = KernRe(r'^\s*/\*\*\s*(@[\w\s]+):\s*(.*)\s*\*/\s*$', cache=False) export_symbol = KernRe(r'^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*', cache=False) export_symbol_ns = KernRe(r'^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*"\S+"\)\s*', cache=False) type_param = KernRe(r"@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) # # Tests for the beginning of a kerneldoc block in its various forms. # doc_block = doc_com + KernRe(r'DOC:\s*(.*)?', cache=False) doc_begin_data = KernRe(r"^\s*\*?\s*(struct|union|enum|typedef)\b\s*(\w*)", cache = False) doc_begin_func = KernRe(str(doc_com) + # initial " * ' r"(?:\w+\s*\*\s*)?" + # type (not captured) r'(?:define\s+)?' + # possible "define" (not captured) r'(\w+)\s*(?:\(\w*\))?\s*' + # name and optional "(...)" r'(?:[-:].*)?$', # description (not captured) cache = False) # # Here begins a long set of transformations to turn structure member prefixes # and macro invocations into something we can parse and generate kdoc for. # struct_args_pattern = r'([^,)]+)' struct_xforms = [ # Strip attributes (KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", flags=re.I | re.S, cache=False), ' '), (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__packed\s*', re.S), ' '), (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), (KernRe(r'\s*____cacheline_aligned', re.S), ' '), (KernRe(r'\s*__cacheline_group_(begin|end)\([^\)]+\);'), ''), # # Unwrap struct_group macros based on this definition: # __struct_group(TAG, NAME, ATTRS, MEMBERS...) # which has variants like: struct_group(NAME, MEMBERS...) # Only MEMBERS arguments require documentation. # # Parsing them happens on two steps: # # 1. drop struct group arguments that aren't at MEMBERS, # storing them as STRUCT_GROUP(MEMBERS) # # 2. remove STRUCT_GROUP() ancillary macro. # # The original logic used to remove STRUCT_GROUP() using an # advanced regex: # # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; # # with two patterns that are incompatible with # Python re module, as it has: # # - a recursive pattern: (?1) # - an atomic grouping: (?>...) # # I tried a simpler version: but it didn't work either: # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; # # As it doesn't properly match the end parenthesis on some cases. # # So, a better solution was crafted: there's now a NestedMatch # class that ensures that delimiters after a search are properly # matched. So, the implementation to drop STRUCT_GROUP() will be # handled in separate. # (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), # # Replace macros # # TODO: use NestedMatch for FOO($1, $2, ...) matches # # it is better to also move those to the NestedMatch logic, # to ensure that parenthesis will be properly matched. # (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), (KernRe(r'DECLARE_BITMAP\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), (KernRe(r'DECLARE_HASHTABLE\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'unsigned long \1[1 << ((\2) - 1)]'), (KernRe(r'DECLARE_KFIFO\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'\1 \2[]'), (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + struct_args_pattern + r'\)', re.S), r'dma_addr_t \1'), (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + struct_args_pattern + r'\)', re.S), r'__u32 \1'), ] # # Regexes here are guaranteed to have the end limiter matching # the start delimiter. Yet, right now, only one replace group # is allowed. # struct_nested_prefixes = [ (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), ] # # Transforms for function prototypes # function_xforms = [ (KernRe(r"^static +"), ""), (KernRe(r"^extern +"), ""), (KernRe(r"^asmlinkage +"), ""), (KernRe(r"^inline +"), ""), (KernRe(r"^__inline__ +"), ""), (KernRe(r"^__inline +"), ""), (KernRe(r"^__always_inline +"), ""), (KernRe(r"^noinline +"), ""), (KernRe(r"^__FORTIFY_INLINE +"), ""), (KernRe(r"__init +"), ""), (KernRe(r"__init_or_module +"), ""), (KernRe(r"__deprecated +"), ""), (KernRe(r"__flatten +"), ""), (KernRe(r"__meminit +"), ""), (KernRe(r"__must_check +"), ""), (KernRe(r"__weak +"), ""), (KernRe(r"__sched +"), ""), (KernRe(r"_noprof"), ""), (KernRe(r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +"), ""), (KernRe(r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +"), ""), (KernRe(r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +"), ""), (KernRe(r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)"), r"\1, \2"), (KernRe(r"__attribute_const__ +"), ""), (KernRe(r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+"), ""), ] # # Apply a set of transforms to a block of text. # def apply_transforms(xforms, text): for search, subst in xforms: text = search.sub(subst, text) return text # # A little helper to get rid of excess white space # multi_space = KernRe(r'\s\s+') def trim_whitespace(s): return multi_space.sub(' ', s.strip()) # # Remove struct/enum members that have been marked "private". # def trim_private_members(text): # # First look for a "public:" block that ends a private region, then # handle the "private until the end" case. # text = KernRe(r'/\*\s*private:.*?/\*\s*public:.*?\*/', flags=re.S).sub('', text) text = KernRe(r'/\*\s*private:.*', flags=re.S).sub('', text) # # We needed the comments to do the above, but now we can take them out. # return KernRe(r'\s*/\*.*?\*/\s*', flags=re.S).sub('', text).strip() class state: """ State machine enums """ # Parser states NORMAL = 0 # normal code NAME = 1 # looking for function name DECLARATION = 2 # We have seen a declaration which might not be done BODY = 3 # the body of the comment SPECIAL_SECTION = 4 # doc section ending with a blank line PROTO = 5 # scanning prototype DOCBLOCK = 6 # documentation block INLINE_NAME = 7 # gathering doc outside main block INLINE_TEXT = 8 # reading the body of inline docs name = [ "NORMAL", "NAME", "DECLARATION", "BODY", "SPECIAL_SECTION", "PROTO", "DOCBLOCK", "INLINE_NAME", "INLINE_TEXT", ] SECTION_DEFAULT = "Description" # default section class KernelEntry: def __init__(self, config, ln): self.config = config self._contents = [] self.prototype = "" self.warnings = [] self.parameterlist = [] self.parameterdescs = {} self.parametertypes = {} self.parameterdesc_start_lines = {} self.section_start_lines = {} self.sections = {} self.anon_struct_union = False self.leading_space = None # State flags self.brcount = 0 self.declaration_start_line = ln + 1 # # Management of section contents # def add_text(self, text): self._contents.append(text) def contents(self): return '\n'.join(self._contents) + '\n' # TODO: rename to emit_message after removal of kernel-doc.pl def emit_msg(self, log_msg, warning=True): """Emit a message""" if not warning: self.config.log.info(log_msg) return # Delegate warning output to output logic, as this way it # will report warnings/info only for symbols that are output self.warnings.append(log_msg) return # # Begin a new section. # def begin_section(self, line_no, title = SECTION_DEFAULT, dump = False): if dump: self.dump_section(start_new = True) self.section = title self.new_start_line = line_no def dump_section(self, start_new=True): """ Dumps section contents to arrays/hashes intended for that purpose. """ # # If we have accumulated no contents in the default ("description") # section, don't bother. # if self.section == SECTION_DEFAULT and not self._contents: return name = self.section contents = self.contents() if type_param.match(name): name = type_param.group(1) self.parameterdescs[name] = contents self.parameterdesc_start_lines[name] = self.new_start_line self.new_start_line = 0 else: if name in self.sections and self.sections[name] != "": # Only warn on user-specified duplicate section names if name != SECTION_DEFAULT: self.emit_msg(self.new_start_line, f"duplicate section name '{name}'\n") # Treat as a new paragraph - add a blank line self.sections[name] += '\n' + contents else: self.sections[name] = contents self.section_start_lines[name] = self.new_start_line self.new_start_line = 0 # self.config.log.debug("Section: %s : %s", name, pformat(vars(self))) if start_new: self.section = SECTION_DEFAULT self._contents = [] class KernelDoc: """ Read a C language source or header FILE and extract embedded documentation comments. """ # Section names section_context = "Context" section_return = "Return" undescribed = "-- undescribed --" def __init__(self, config, fname): """Initialize internal variables""" self.fname = fname self.config = config # Initial state for the state machines self.state = state.NORMAL # Store entry currently being processed self.entry = None # Place all potential outputs into an array self.entries = [] # # We need Python 3.7 for its "dicts remember the insertion # order" guarantee # if sys.version_info.major == 3 and sys.version_info.minor < 7: self.emit_msg(0, 'Python 3.7 or later is required for correct results') def emit_msg(self, ln, msg, warning=True): """Emit a message""" log_msg = f"{self.fname}:{ln} {msg}" if self.entry: self.entry.emit_msg(log_msg, warning) return if warning: self.config.log.warning(log_msg) else: self.config.log.info(log_msg) def dump_section(self, start_new=True): """ Dumps section contents to arrays/hashes intended for that purpose. """ if self.entry: self.entry.dump_section(start_new) # TODO: rename it to store_declaration after removal of kernel-doc.pl def output_declaration(self, dtype, name, **args): """ Stores the entry into an entry array. The actual output and output filters will be handled elsewhere """ item = KdocItem(name, dtype, self.entry.declaration_start_line, **args) item.warnings = self.entry.warnings # Drop empty sections # TODO: improve empty sections logic to emit warnings sections = self.entry.sections for section in ["Description", "Return"]: if section in sections and not sections[section].rstrip(): del sections[section] item.set_sections(sections, self.entry.section_start_lines) item.set_params(self.entry.parameterlist, self.entry.parameterdescs, self.entry.parametertypes, self.entry.parameterdesc_start_lines) self.entries.append(item) self.config.log.debug("Output: %s:%s = %s", dtype, name, pformat(args)) def reset_state(self, ln): """ Ancillary routine to create a new entry. It initializes all variables used by the state machine. """ self.entry = KernelEntry(self.config, ln) # State flags self.state = state.NORMAL def push_parameter(self, ln, decl_type, param, dtype, org_arg, declaration_name): """ Store parameters and their descriptions at self.entry. """ if self.entry.anon_struct_union and dtype == "" and param == "}": return # Ignore the ending }; from anonymous struct/union self.entry.anon_struct_union = False param = KernRe(r'[\[\)].*').sub('', param, count=1) # # Look at various "anonymous type" cases. # if dtype == '': if param.endswith("..."): if len(param) > 3: # there is a name provided, use that param = param[:-3] if not self.entry.parameterdescs.get(param): self.entry.parameterdescs[param] = "variable arguments" elif (not param) or param == "void": param = "void" self.entry.parameterdescs[param] = "no arguments" elif param in ["struct", "union"]: # Handle unnamed (anonymous) union or struct dtype = param param = "{unnamed_" + param + "}" self.entry.parameterdescs[param] = "anonymous\n" self.entry.anon_struct_union = True # Warn if parameter has no description # (but ignore ones starting with # as these are not parameters # but inline preprocessor statements) if param not in self.entry.parameterdescs and not param.startswith("#"): self.entry.parameterdescs[param] = self.undescribed if "." not in param: if decl_type == 'function': dname = f"{decl_type} parameter" else: dname = f"{decl_type} member" self.emit_msg(ln, f"{dname} '{param}' not described in '{declaration_name}'") # Strip spaces from param so that it is one continuous string on # parameterlist. This fixes a problem where check_sections() # cannot find a parameter like "addr[6 + 2]" because it actually # appears as "addr[6", "+", "2]" on the parameter list.