docs: kernel-doc: add support to store output on a YAML file

Add a command line parameter and library support to optionally store: - KdocItem intermediate format after parsing; - man pages output; - rst output. inside a YAML file.

Mauro Carvalho Chehab committed 7 months ago

Commit 01d6d7b1 parent b37b3cb3 files · +107 −15Browse files

Showing the changes to tools/lib/python (2 of 3 files). Show all

Unified · Split

2 files

tools/lib/python/kdoc/kdoc_files.py +41 −6
tools/lib/python/kdoc/kdoc_parser.py +26 −1

tools/lib/python/kdoc/kdoc_files.py+41−6

 from kdoc.kdoc_parser import KernelDoc
 from kdoc.xforms_lists import CTransforms
 from kdoc.kdoc_output import OutputFormat
+from kdoc.kdoc_yaml_file import KDocTestFile
 class GlobSourceFiles:
         If not specified, defaults to use: ``logging.getLogger("kernel-doc")``
+    ``yaml_file``
+        If defined, stores the output inside a YAML file.
+    ``yaml_content``
+        Defines what will be inside the YAML file.
     Note:
         There are two type of parsers defined here:
         if fname in self.files:
             return
-        doc = KernelDoc(self.config, fname, self.xforms)
+        if self.test_file:
+            store_src = True
+        else:
+            store_src = False
+        doc = KernelDoc(self.config, fname, self.xforms, store_src=store_src)
         export_table, entries = doc.parse_kdoc()
         self.export_table[fname] = export_table
         self.results[fname] = entries
+        source = doc.get_source()
+        if source:
+            self.source[fname] = source
     def process_export_file(self, fname):
         """
         Parses ``EXPORT_SYMBOL*`` macros from a single Kernel source file.
     def __init__(self, verbose=False, out_style=None, xforms=None,
                  werror=False, wreturn=False, wshort_desc=False,
                  wcontents_before_sections=False,
-                 logger=None):
+                 yaml_file=None, yaml_content=None, logger=None):
         """
         Initialize startup variables and parse all files.
         """
         # Override log warning, as we want to count errors
         self.config.warning = self.warning
+        if yaml_file:
+            self.test_file = KDocTestFile(self.config, yaml_file, yaml_content)
+        else:
+            self.test_file = None
         if xforms:
             self.xforms = xforms
         else:
         self.errors = 0
         self.results = {}
+        self.source = {}
         self.files = set()
         self.export_files = set()
                 for s in symbol:
                     function_table.add(s)
-            self.out_style.set_filter(export, internal, symbol, nosymbol,
-                                      function_table, enable_lineno,
-                                      no_doc_sections)
             if fname not in self.results:
                 self.config.log.warning("No kernel-doc for file %s", fname)
                 continue
             symbols = self.results[fname]
+            if self.test_file:
+                self.test_file.set_filter(export, internal, symbol, nosymbol,
+                                          function_table, enable_lineno,
+                                          no_doc_sections)
+                self.test_file.output_symbols(fname, symbols,
+                                              self.source.get(fname))
+                continue
+            self.out_style.set_filter(export, internal, symbol, nosymbol,
+                                      function_table, enable_lineno,
+                                      no_doc_sections)
             msg = self.out_style.output_symbols(fname, symbols)
             if msg:
                 yield fname, msg
+        if self.test_file:
+            self.test_file.write()

tools/lib/python/kdoc/kdoc_parser.py+26−1

     #: String to write when a parameter is not described.
     undescribed = "-- undescribed --"
-    def __init__(self, config, fname, xforms):
+    def __init__(self, config, fname, xforms, store_src=False):
         """Initialize internal variables"""
         self.fname = fname
         self.config = config
         self.xforms = xforms
+        self.store_src = store_src
         tokenizer_set_log(self.config.log, f"{self.fname}: CMatch: ")
         # Place all potential outputs into an array
         self.entries = []
+        # When store_src is true, the kernel-doc source content is stored here
+        self.source = None
         #
         # We need Python 3.7 for its "dicts remember the insertion
         # order" guarantee
         state.DOCBLOCK:			process_docblock,
         }
+    def get_source(self):
+        """
+        Return the file content of the lines handled by kernel-doc at the
+        latest parse_kdoc() run.
+        Returns none if KernelDoc() was not initialized with store_src,
+        """
+        return self.source
     def parse_kdoc(self):
         """
         Open and process each line of a C source file.
         prev = ""
         prev_ln = None
         export_table = set()
+        self.source = []
+        self.state = state.NORMAL
         try:
             with open(self.fname, "r", encoding="utf8",
                                           ln, state.name[self.state],
                                           line)
+                    prev_state = self.state
                     # This is an optimization over the original script.
                     # There, when export_file was used for the same file,
                     # it was read twice. Here, we use the already-existing
                         # Hand this line to the appropriate state handler
                         self.state_actions[self.state](self, ln, line)
+                    if self.store_src and prev_state != self.state or self.state != state.NORMAL:
+                        if self.state == state.NAME:
+                            # A "/**" was detected. Add a new source element
+                            self.source.append({"ln": ln, "data": line + "\n"})
+                        else:
+                            # Append to the existing one
+                            self.source[-1]["data"] += line + "\n"
             self.emit_unused_warnings()
         except OSError: