#!/usr/bin/env python3# SPDX-License-Identifier: GPL-2.0# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>.## pylint: disable=C0301,C0302,R0904,R0912,R0913,R0914,R0915,R0917,R1702 """Classes and functions related to reading a C language source or header FILEand extract embedded documentation comments from it.""" import sysimport refrom pprint import pformat from kdoc.kdoc_re import NestedMatch, KernRefrom kdoc.kdoc_item import KdocItem ## Regular expressions used to parse kernel-doc markups at KernelDoc class.## Let's declare them in lowercase outside any class to make it easier to# convert from the Perl script.## As those are evaluated at the beginning, no need to cache them# # Allow whitespace at end of comment start.doc_start = KernRe(r'^/\*\*\s*$', cache=False) doc_end = KernRe(r'\*/', cache=False)doc_com = KernRe(r'\s*\*\s*', cache=False)doc_com_body = KernRe(r'\s*\* ?', cache=False)doc_decl = doc_com + KernRe(r'(\w+)', cache=False) # @params and a strictly limited set of supported section names# Specifically:# Match @word:# @...:# @{section-name}:# while trying to not match literal block starts like "example::"#known_section_names = 'description|context|returns?|notes?|examples?'known_sections = KernRe(known_section_names, flags = re.I)doc_sect = doc_com + \ KernRe(r'\s*(@[.\w]+|@\.\.\.|' + known_section_names + r')\s*:([^:].*)?$', flags=re.I, cache=False) doc_content = doc_com_body + KernRe(r'(.*)', cache=False)doc_inline_start = KernRe(r'^\s*/\*\*\s*$', cache=False)doc_inline_sect = KernRe(r'\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)', cache=False)doc_inline_end = KernRe(r'^\s*\*/\s*$', cache=False)doc_inline_oneline = KernRe(r'^\s*/\*\*\s*(@\s*[\w][\w\.]*\s*):\s*(.*)\s*\*/\s*$', cache=False) export_symbol = KernRe(r'^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*', cache=False)export_symbol_ns = KernRe(r'^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*"\S+"\)\s*', cache=False) type_param = KernRe(r"@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) ## Tests for the beginning of a kerneldoc block in its various forms.#doc_block = doc_com + KernRe(r'DOC:\s*(.*)?', cache=False)doc_begin_data = KernRe(r"^\s*\*?\s*(struct|union|enum|typedef|var)\b\s*(\w*)", cache = False)doc_begin_func = KernRe(str(doc_com) + # initial " * ' r"(?:\w+\s*\*\s*)?" + # type (not captured) r'(?:define\s+)?' + # possible "define" (not captured) r'(\w+)\s*(?:\(\w*\))?\s*' + # name and optional "(...)" r'(?:[-:].*)?$', # description (not captured) cache = False) ## Here begins a long set of transformations to turn structure member prefixes# and macro invocations into something we can parse and generate kdoc for.#struct_args_pattern = r'([^,)]+)' struct_xforms = [ # Strip attributes (KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", flags=re.I | re.S, cache=False), ' '), (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__packed\s*', re.S), ' '), (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), (KernRe(r'\s*__private', re.S), ' '), (KernRe(r'\s*__rcu', re.S), ' '), (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), (KernRe(r'\s*____cacheline_aligned', re.S), ' '), (KernRe(r'\s*__cacheline_group_(begin|end)\([^\)]+\);'), ''), # # Unwrap struct_group macros based on this definition: # __struct_group(TAG, NAME, ATTRS, MEMBERS...) # which has variants like: struct_group(NAME, MEMBERS...) # Only MEMBERS arguments require documentation. # # Parsing them happens on two steps: # # 1. drop struct group arguments that aren't at MEMBERS, # storing them as STRUCT_GROUP(MEMBERS) # # 2. remove STRUCT_GROUP() ancillary macro. # # The original logic used to remove STRUCT_GROUP() using an # advanced regex: # # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; # # with two patterns that are incompatible with # Python re module, as it has: # # - a recursive pattern: (?1) # - an atomic grouping: (?>...) # # I tried a simpler version: but it didn't work either: # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; # # As it doesn't properly match the end parenthesis on some cases. # # So, a better solution was crafted: there's now a NestedMatch # class that ensures that delimiters after a search are properly # matched. So, the implementation to drop STRUCT_GROUP() will be # handled in separate. # (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), # # Replace macros # # TODO: use NestedMatch for FOO($1, $2, ...) matches # # it is better to also move those to the NestedMatch logic, # to ensure that parentheses will be properly matched. # (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), (KernRe(r'DECLARE_BITMAP\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), (KernRe(r'DECLARE_HASHTABLE\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'unsigned long \1[1 << ((\2) - 1)]'), (KernRe(r'DECLARE_KFIFO\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'\1 \2[]'), (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + struct_args_pattern + r'\)', re.S), r'dma_addr_t \1'), (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + struct_args_pattern + r'\)', re.S), r'__u32 \1'),]## Regexes here are guaranteed to have the end delimiter matching# the start delimiter. Yet, right now, only one replace group# is allowed.#struct_nested_prefixes = [ (re.compile(r'\bSTRUCT_GROUP\('), r'\1'),] ## Transforms for function prototypes#function_xforms = [ (KernRe(r"^static +"), ""), (KernRe(r"^extern +"), ""), (KernRe(r"^asmlinkage +"), ""), (KernRe(r"^inline +"), ""), (KernRe(r"^__inline__ +"), ""), (KernRe(r"^__inline +"), ""), (KernRe(r"^__always_inline +"), ""), (KernRe(r"^noinline +"), ""), (KernRe(r"^__FORTIFY_INLINE +"), ""), (KernRe(r"__init +"), ""), (KernRe(r"__init_or_module +"), ""), (KernRe(r"__exit +"), ""), (KernRe(r"__deprecated +"), ""), (KernRe(r"__flatten +"), ""), (KernRe(r"__meminit +"), ""), (KernRe(r"__must_check +"), ""), (KernRe(r"__weak +"), ""), (KernRe(r"__sched +"), ""), (KernRe(r"_noprof"), ""), (KernRe(r"__always_unused *"), ""), (KernRe(r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +"), ""), (KernRe(r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +"), ""), (KernRe(r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +"), ""), (KernRe(r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)"), r"\1, \2"), (KernRe(r"__attribute_const__ +"), ""), (KernRe(r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+"), ""),] ## Ancillary functions# def apply_transforms(xforms, text): """ Apply a set of transforms to a block of text. """ for search, subst in xforms: text = search.sub(subst, text) return text multi_space = KernRe(r'\s\s+')def trim_whitespace(s): """ A little helper to get rid of excess white space. """ return multi_space.sub(' ', s.strip()) def trim_private_members(text): """ Remove ``struct``/``enum`` members that have been marked "private". """ # First look for a "public:" block that ends a private region, then # handle the "private until the end" case. # text = KernRe(r'/\*\s*private:.*?/\*\s*public:.*?\*/', flags=re.S).sub('', text) text = KernRe(r'/\*\s*private:.*', flags=re.S).sub('', text) # # We needed the comments to do the above, but now we can take them out. # return KernRe(r'\s*/\*.*?\*/\s*', flags=re.S).sub('', text).strip() class state: """ States used by the parser's state machine. """ # Parser states NORMAL = 0 #: Normal code. NAME = 1 #: Looking for function name. DECLARATION = 2 #: We have seen a declaration which might not be done. BODY = 3 #: The body of the comment. SPECIAL_SECTION = 4 #: Doc section ending with a blank line. PROTO = 5 #: Scanning prototype. DOCBLOCK = 6 #: Documentation block. INLINE_NAME = 7 #: Gathering doc outside main block. INLINE_TEXT = 8 #: Reading the body of inline docs. #: Names for each parser state. name = [ "NORMAL", "NAME", "DECLARATION", "BODY", "SPECIAL_SECTION", "PROTO", "DOCBLOCK", "INLINE_NAME", "INLINE_TEXT", ] SECTION_DEFAULT = "Description" #: Default section. class KernelEntry: """ Encapsulates a Kernel documentation entry. """ def __init__(self, config, fname, ln): self.config = config self.fname = fname self._contents = [] self.prototype = "" self.warnings = [] self.parameterlist = [] self.parameterdescs = {} self.parametertypes = {} self.parameterdesc_start_lines = {} self.section_start_lines = {} self.sections = {} self.anon_struct_union = False self.leading_space = None self.fname = fname # State flags self.brcount = 0 self.declaration_start_line = ln + 1 # # Management of section contents # def add_text(self, text): """Add a new text to the entry contents list.""" self._contents.append(text) def contents(self): """Returns a string with all content texts that were added.""" return '\n'.join(self._contents) + '\n' # TODO: rename to emit_message after removal of kernel-doc.pl def emit_msg(self, ln, msg, *, warning=True): """Emit a message.""" log_msg = f"{self.fname}:{ln} {msg}" if not warning: self.config.log.info(log_msg) return # Delegate warning output to output logic, as this way it # will report warnings/info only for symbols that are output self.warnings.append(log_msg) return def begin_section(self, line_no, title = SECTION_DEFAULT, dump = False): """ Begin a new section. """ if dump: self.dump_section(start_new = True) self.section = title self.new_start_line = line_no def dump_section(self, start_new=True): """ Dumps section contents to arrays/hashes intended for that purpose. """ # # If we have accumulated no contents in the default ("description") # section, don't bother. # if self.section == SECTION_DEFAULT and not self._contents: return name = self.section contents = self.contents() if type_param.match(name): name = type_param.group(1) self.parameterdescs[name] = contents self.parameterdesc_start_lines[name] = self.new_start_line self.new_start_line = 0 else: if name in self.sections and self.sections[name] != "": # Only warn on user-specified duplicate section names if name != SECTION_DEFAULT: self.emit_msg(self.new_start_line, f"duplicate section name '{name}'") # Treat as a new paragraph - add a blank line self.sections[name] += '\n' + contents else: self.sections[name] = contents self.section_start_lines[name] = self.new_start_line self.new_start_line = 0 # self.config.log.debug("Section: %s : %s", name, pformat(vars(self))) if start_new: self.section = SECTION_DEFAULT self._contents = [] python_warning = False class KernelDoc: """ Read a C language source or header FILE and extract embedded documentation comments. """ #: Name of context section. section_context = "Context" #: Name of return section. section_return = "Return" #: String to write when a parameter is not described. undescribed = "-- undescribed --" def __init__(self, config, fname): """Initialize internal variables""" self.fname = fname self.config = config # Initial state for the state machines self.state = state.NORMAL # Store entry currently being processed self.entry = None # Place all potential outputs into an array self.entries = [] # # We need Python 3.7 for its "dicts remember the insertion # order" guarantee # global python_warning if (not python_warning and sys.version_info.major == 3 and sys.version_info.minor < 7): self.emit_msg(0, 'Python 3.7 or later is required for correct results') python_warning = True def emit_msg(self, ln, msg, *, warning=True): """Emit a message""" if self.entry: self.entry.emit_msg(ln, msg, warning=warning) return log_msg = f"{self.fname}:{ln} {msg}" if warning: self.config.log.warning(log_msg) else: self.config.log.info(log_msg) def dump_section(self, start_new=True): """ Dump section contents to arrays/hashes intended for that purpose. """ if self.entry: self.entry.dump_section(start_new) # TODO: rename it to store_declaration after removal of kernel-doc.pl def output_declaration(self, dtype, name, **args): """ Store the entry into an entry array. The actual output and output filters will be handled elsewhere. """ item = KdocItem(name, self.fname, dtype, self.entry.declaration_start_line, **args) item.warnings = self.entry.warnings # Drop empty sections # TODO: improve empty sections logic to emit warnings sections = self.entry.sections for section in ["Description", "Return"]: if section in sections and not sections[section].rstrip(): del sections[section] item.set_sections(sections, self.entry.section_start_lines) item.set_params(self.entry.parameterlist, self.entry.parameterdescs, self.entry.parametertypes, self.entry.parameterdesc_start_lines) self.entries.append(item) self.config.log.debug("Output: %s:%s = %s", dtype, name, pformat(args)) def emit_unused_warnings(self): """ When the parser fails to produce a valid entry, it places some warnings under `entry.warnings` that will be discarded when resetting the state. Ensure that those warnings are not lost. .. note:: Because we are calling `config.warning()` here, those warnings are not filtered by the `-W` parameters: they will all be produced even when `-Wreturn`, `-Wshort-desc`, and/or `-Wcontents-before-sections` are used. Allowing those warnings to be filtered is complex, because it would require storing them in a buffer and then filtering them during the output step of the code, depending on the selected symbols. """ if self.entry and self.entry not in self.entries: for log_msg in self.entry.warnings: self.config.warning(log_msg) def reset_state(self, ln): """ Ancillary routine to create a new entry. It initializes all variables used by the state machine. """ self.emit_unused_warnings() self.entry = KernelEntry(self.config, self.fname, ln) # State flags self.state = state.NORMAL def push_parameter(self, ln, decl_type, param, dtype, org_arg, declaration_name): """ Store parameters and their descriptions at self.entry.Lines 1–500 of 1784. Load more
#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 # Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. # # pylint: disable=C0301,C0302,R0904,R0912,R0913,R0914,R0915,R0917,R1702 """ Classes and functions related to reading a C language source or header FILE and extract embedded documentation comments from it. """ import sys import re from pprint import pformat from kdoc.kdoc_re import NestedMatch, KernRe from kdoc.kdoc_item import KdocItem # # Regular expressions used to parse kernel-doc markups at KernelDoc class. # # Let's declare them in lowercase outside any class to make it easier to # convert from the Perl script. # # As those are evaluated at the beginning, no need to cache them # # Allow whitespace at end of comment start. doc_start = KernRe(r'^/\*\*\s*$', cache=False) doc_end = KernRe(r'\*/', cache=False) doc_com = KernRe(r'\s*\*\s*', cache=False) doc_com_body = KernRe(r'\s*\* ?', cache=False) doc_decl = doc_com + KernRe(r'(\w+)', cache=False) # @params and a strictly limited set of supported section names # Specifically: # Match @word: # @...: # @{section-name}: # while trying to not match literal block starts like "example::" # known_section_names = 'description|context|returns?|notes?|examples?' known_sections = KernRe(known_section_names, flags = re.I) doc_sect = doc_com + \ KernRe(r'\s*(@[.\w]+|@\.\.\.|' + known_section_names + r')\s*:([^:].*)?$', flags=re.I, cache=False) doc_content = doc_com_body + KernRe(r'(.*)', cache=False) doc_inline_start = KernRe(r'^\s*/\*\*\s*$', cache=False) doc_inline_sect = KernRe(r'\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)', cache=False) doc_inline_end = KernRe(r'^\s*\*/\s*$', cache=False) doc_inline_oneline = KernRe(r'^\s*/\*\*\s*(@\s*[\w][\w\.]*\s*):\s*(.*)\s*\*/\s*$', cache=False) export_symbol = KernRe(r'^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*', cache=False) export_symbol_ns = KernRe(r'^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*"\S+"\)\s*', cache=False) type_param = KernRe(r"@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) # # Tests for the beginning of a kerneldoc block in its various forms. # doc_block = doc_com + KernRe(r'DOC:\s*(.*)?', cache=False) doc_begin_data = KernRe(r"^\s*\*?\s*(struct|union|enum|typedef|var)\b\s*(\w*)", cache = False) doc_begin_func = KernRe(str(doc_com) + # initial " * ' r"(?:\w+\s*\*\s*)?" + # type (not captured) r'(?:define\s+)?' + # possible "define" (not captured) r'(\w+)\s*(?:\(\w*\))?\s*' + # name and optional "(...)" r'(?:[-:].*)?$', # description (not captured) cache = False) # # Here begins a long set of transformations to turn structure member prefixes # and macro invocations into something we can parse and generate kdoc for. # struct_args_pattern = r'([^,)]+)' struct_xforms = [ # Strip attributes (KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", flags=re.I | re.S, cache=False), ' '), (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__packed\s*', re.S), ' '), (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), (KernRe(r'\s*__private', re.S), ' '), (KernRe(r'\s*__rcu', re.S), ' '), (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), (KernRe(r'\s*____cacheline_aligned', re.S), ' '), (KernRe(r'\s*__cacheline_group_(begin|end)\([^\)]+\);'), ''), # # Unwrap struct_group macros based on this definition: # __struct_group(TAG, NAME, ATTRS, MEMBERS...) # which has variants like: struct_group(NAME, MEMBERS...) # Only MEMBERS arguments require documentation. # # Parsing them happens on two steps: # # 1. drop struct group arguments that aren't at MEMBERS, # storing them as STRUCT_GROUP(MEMBERS) # # 2. remove STRUCT_GROUP() ancillary macro. # # The original logic used to remove STRUCT_GROUP() using an # advanced regex: # # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; # # with two patterns that are incompatible with # Python re module, as it has: # # - a recursive pattern: (?1) # - an atomic grouping: (?>...) # # I tried a simpler version: but it didn't work either: # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; # # As it doesn't properly match the end parenthesis on some cases. # # So, a better solution was crafted: there's now a NestedMatch # class that ensures that delimiters after a search are properly # matched. So, the implementation to drop STRUCT_GROUP() will be # handled in separate. # (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), # # Replace macros # # TODO: use NestedMatch for FOO($1, $2, ...) matches # # it is better to also move those to the NestedMatch logic, # to ensure that parentheses will be properly matched. # (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), (KernRe(r'DECLARE_BITMAP\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), (KernRe(r'DECLARE_HASHTABLE\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'unsigned long \1[1 << ((\2) - 1)]'), (KernRe(r'DECLARE_KFIFO\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', re.S), r'\1 \2[]'), (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + struct_args_pattern + r'\)', re.S), r'dma_addr_t \1'), (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + struct_args_pattern + r'\)', re.S), r'__u32 \1'), ] # # Regexes here are guaranteed to have the end delimiter matching # the start delimiter. Yet, right now, only one replace group # is allowed. # struct_nested_prefixes = [ (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), ] # # Transforms for function prototypes # function_xforms = [ (KernRe(r"^static +"), ""), (KernRe(r"^extern +"), ""), (KernRe(r"^asmlinkage +"), ""), (KernRe(r"^inline +"), ""), (KernRe(r"^__inline__ +"), ""), (KernRe(r"^__inline +"), ""), (KernRe(r"^__always_inline +"), ""), (KernRe(r"^noinline +"), ""), (KernRe(r"^__FORTIFY_INLINE +"), ""), (KernRe(r"__init +"), ""), (KernRe(r"__init_or_module +"), ""), (KernRe(r"__exit +"), ""), (KernRe(r"__deprecated +"), ""), (KernRe(r"__flatten +"), ""), (KernRe(r"__meminit +"), ""), (KernRe(r"__must_check +"), ""), (KernRe(r"__weak +"), ""), (KernRe(r"__sched +"), ""), (KernRe(r"_noprof"), ""), (KernRe(r"__always_unused *"), ""), (KernRe(r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +"), ""), (KernRe(r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +"), ""), (KernRe(r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +"), ""), (KernRe(r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)"), r"\1, \2"), (KernRe(r"__attribute_const__ +"), ""), (KernRe(r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+"), ""), ] # # Ancillary functions # def apply_transforms(xforms, text): """ Apply a set of transforms to a block of text. """ for search, subst in xforms: text = search.sub(subst, text) return text multi_space = KernRe(r'\s\s+') def trim_whitespace(s): """ A little helper to get rid of excess white space. """ return multi_space.sub(' ', s.strip()) def trim_private_members(text): """ Remove ``struct``/``enum`` members that have been marked "private". """ # First look for a "public:" block that ends a private region, then # handle the "private until the end" case. # text = KernRe(r'/\*\s*private:.*?/\*\s*public:.*?\*/', flags=re.S).sub('', text) text = KernRe(r'/\*\s*private:.*', flags=re.S).sub('', text) # # We needed the comments to do the above, but now we can take them out. # return KernRe(r'\s*/\*.*?\*/\s*', flags=re.S).sub('', text).strip() class state: """ States used by the parser's state machine. """ # Parser states NORMAL = 0 #: Normal code. NAME = 1 #: Looking for function name. DECLARATION = 2 #: We have seen a declaration which might not be done. BODY = 3 #: The body of the comment. SPECIAL_SECTION = 4 #: Doc section ending with a blank line. PROTO = 5 #: Scanning prototype. DOCBLOCK = 6 #: Documentation block. INLINE_NAME = 7 #: Gathering doc outside main block. INLINE_TEXT = 8 #: Reading the body of inline docs. #: Names for each parser state. name = [ "NORMAL", "NAME", "DECLARATION", "BODY", "SPECIAL_SECTION", "PROTO", "DOCBLOCK", "INLINE_NAME", "INLINE_TEXT", ] SECTION_DEFAULT = "Description" #: Default section. class KernelEntry: """ Encapsulates a Kernel documentation entry. """ def __init__(self, config, fname, ln): self.config = config self.fname = fname self._contents = [] self.prototype = "" self.warnings = [] self.parameterlist = [] self.parameterdescs = {} self.parametertypes = {} self.parameterdesc_start_lines = {} self.section_start_lines = {} self.sections = {} self.anon_struct_union = False self.leading_space = None self.fname = fname # State flags self.brcount = 0 self.declaration_start_line = ln + 1 # # Management of section contents # def add_text(self, text): """Add a new text to the entry contents list.""" self._contents.append(text) def contents(self): """Returns a string with all content texts that were added.""" return '\n'.join(self._contents) + '\n' # TODO: rename to emit_message after removal of kernel-doc.pl def emit_msg(self, ln, msg, *, warning=True): """Emit a message.""" log_msg = f"{self.fname}:{ln} {msg}" if not warning: self.config.log.info(log_msg) return # Delegate warning output to output logic, as this way it # will report warnings/info only for symbols that are output self.warnings.append(log_msg) return def begin_section(self, line_no, title = SECTION_DEFAULT, dump = False): """ Begin a new section. """ if dump: self.dump_section(start_new = True) self.section = title self.new_start_line = line_no def dump_section(self, start_new=True): """ Dumps section contents to arrays/hashes intended for that purpose. """ # # If we have accumulated no contents in the default ("description") # section, don't bother. # if self.section == SECTION_DEFAULT and not self._contents: return name = self.section contents = self.contents() if type_param.match(name): name = type_param.group(1) self.parameterdescs[name] = contents self.parameterdesc_start_lines[name] = self.new_start_line self.new_start_line = 0 else: if name in self.sections and self.sections[name] != "": # Only warn on user-specified duplicate section names if name != SECTION_DEFAULT: self.emit_msg(self.new_start_line, f"duplicate section name '{name}'") # Treat as a new paragraph - add a blank line self.sections[name] += '\n' + contents else: self.sections[name] = contents self.section_start_lines[name] = self.new_start_line self.new_start_line = 0 # self.config.log.debug("Section: %s : %s", name, pformat(vars(self))) if start_new: self.section = SECTION_DEFAULT self._contents = [] python_warning = False class KernelDoc: """ Read a C language source or header FILE and extract embedded documentation comments. """ #: Name of context section. section_context = "Context" #: Name of return section. section_return = "Return" #: String to write when a parameter is not described. undescribed = "-- undescribed --" def __init__(self, config, fname): """Initialize internal variables""" self.fname = fname self.config = config # Initial state for the state machines self.state = state.NORMAL # Store entry currently being processed self.entry = None # Place all potential outputs into an array self.entries = [] # # We need Python 3.7 for its "dicts remember the insertion # order" guarantee # global python_warning if (not python_warning and sys.version_info.major == 3 and sys.version_info.minor < 7): self.emit_msg(0, 'Python 3.7 or later is required for correct results') python_warning = True def emit_msg(self, ln, msg, *, warning=True): """Emit a message""" if self.entry: self.entry.emit_msg(ln, msg, warning=warning) return log_msg = f"{self.fname}:{ln} {msg}" if warning: self.config.log.warning(log_msg) else: self.config.log.info(log_msg) def dump_section(self, start_new=True): """ Dump section contents to arrays/hashes intended for that purpose. """ if self.entry: self.entry.dump_section(start_new) # TODO: rename it to store_declaration after removal of kernel-doc.pl def output_declaration(self, dtype, name, **args): """ Store the entry into an entry array. The actual output and output filters will be handled elsewhere. """ item = KdocItem(name, self.fname, dtype, self.entry.declaration_start_line, **args) item.warnings = self.entry.warnings # Drop empty sections # TODO: improve empty sections logic to emit warnings sections = self.entry.sections for section in ["Description", "Return"]: if section in sections and not sections[section].rstrip(): del sections[section] item.set_sections(sections, self.entry.section_start_lines) item.set_params(self.entry.parameterlist, self.entry.parameterdescs, self.entry.parametertypes, self.entry.parameterdesc_start_lines) self.entries.append(item) self.config.log.debug("Output: %s:%s = %s", dtype, name, pformat(args)) def emit_unused_warnings(self): """ When the parser fails to produce a valid entry, it places some warnings under `entry.warnings` that will be discarded when resetting the state. Ensure that those warnings are not lost. .. note:: Because we are calling `config.warning()` here, those warnings are not filtered by the `-W` parameters: they will all be produced even when `-Wreturn`, `-Wshort-desc`, and/or `-Wcontents-before-sections` are used. Allowing those warnings to be filtered is complex, because it would require storing them in a buffer and then filtering them during the output step of the code, depending on the selected symbols. """ if self.entry and self.entry not in self.entries: for log_msg in self.entry.warnings: self.config.warning(log_msg) def reset_state(self, ln): """ Ancillary routine to create a new entry. It initializes all variables used by the state machine. """ self.emit_unused_warnings() self.entry = KernelEntry(self.config, self.fname, ln) # State flags self.state = state.NORMAL def push_parameter(self, ln, decl_type, param, dtype, org_arg, declaration_name): """ Store parameters and their descriptions at self.entry.