History

scripts/kernel-doc.py: properly handle struct_group macros

Handing nested parenthesis with regular expressions is not an easy task. It is even harder with Python's re module, as it has a limited subset of regular expressions, missing more advanced features. We might use instead Python regex module, but still the regular expressions are very hard to understand. So, instead, add a logic to properly match delimiters.

Mauro Carvalho Chehab committed 1 year ago

Commit 01d32351 parent 35923851 file · +213 −7Browse files

Unified · Split

scripts/kernel-doc.py+213−7
@@ -167,6 +167,172 @@ class Re: · 166 unmodified lines
167167 def group(self, num):
168168 return self.last_match.group(num)
169169
170+class NestedMatch:
171+ """
172+ Finding nested delimiters is hard with regular expressions. It is
173+ even harder on Python with its normal re module, as there are several
174+ advanced regular expressions that are missing.
175+
176+ This is the case of this pattern:
177+
178+ '\\bSTRUCT_GROUP(\\(((?:(?>[^)(]+)|(?1))*)\\))[^;]*;'
179+
180+ which is used to properly match open/close parenthesis of the
181+ string search STRUCT_GROUP(),
182+
183+ Add a class that counts pairs of delimiters, using it to match and
184+ replace nested expressions.
185+
186+ The original approach was suggested by:
187+ https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex
188+
189+ Although I re-implemented it to make it more generic and match 3 types
190+ of delimiters. The logic checks if delimiters are paired. If not, it
191+ will ignore the search string.
192+ """
193+
194+ # TODO:
195+ # Right now, regular expressions to match it are defined only up to
196+ # the start delimiter, e.g.:
197+ #
198+ # \bSTRUCT_GROUP\(
199+ #
200+ # is similar to: STRUCT_GROUP\((.*)\)
201+ # except that the content inside the match group is delimiter's aligned.
202+ #
203+ # The content inside parenthesis are converted into a single replace
204+ # group (e.g. r`\1').
205+ #
206+ # It would be nice to change such definition to support multiple
207+ # match groups, allowing a regex equivalent to.
208+ #
209+ # FOO\((.*), (.*), (.*)\)
210+ #
211+ # it is probably easier to define it not as a regular expression, but
212+ # with some lexical definition like:
213+ #
214+ # FOO(arg1, arg2, arg3)
215+
216+
217+ DELIMITER_PAIRS = {
218+ '{': '}',
219+ '(': ')',
220+ '[': ']',
221+ }
222+
223+ RE_DELIM = re.compile(r'[\{\}\[\]\(\)]')
224+
225+ def _search(self, regex, line):
226+ """
227+ Finds paired blocks for a regex that ends with a delimiter.
228+
229+ The suggestion of using finditer to match pairs came from:
230+ https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex
231+ but I ended using a different implementation to align all three types
232+ of delimiters and seek for an initial regular expression.
233+
234+ The algorithm seeks for open/close paired delimiters and place them
235+ into a stack, yielding a start/stop position of each match when the
236+ stack is zeroed.
237+
238+ The algorithm shoud work fine for properly paired lines, but will
239+ silently ignore end delimiters that preceeds an start delimiter.
240+ This should be OK for kernel-doc parser, as unaligned delimiters
241+ would cause compilation errors. So, we don't need to rise exceptions
242+ to cover such issues.
243+ """
244+
245+ stack = []
246+
247+ for match_re in regex.finditer(line):
248+ start = match_re.start()
249+ offset = match_re.end()
250+
251+ d = line[offset -1]
252+ if d not in self.DELIMITER_PAIRS:
253+ continue
254+
255+ end = self.DELIMITER_PAIRS[d]
256+ stack.append(end)
257+
258+ for match in self.RE_DELIM.finditer(line[offset:]):
259+ pos = match.start() + offset
260+
261+ d = line[pos]
262+
263+ if d in self.DELIMITER_PAIRS:
264+ end = self.DELIMITER_PAIRS[d]
265+
266+ stack.append(end)
267+ continue
268+
269+ # Does the end delimiter match what it is expected?
270+ if stack and d == stack[-1]:
271+ stack.pop()
272+
273+ if not stack:
274+ yield start, offset, pos + 1
275+ break
276+
277+ def search(self, regex, line):
278+ """
279+ This is similar to re.search:
280+
281+ It matches a regex that it is followed by a delimiter,
282+ returning occurrences only if all delimiters are paired.
283+ """
284+
285+ for t in self._search(regex, line):
286+
287+ yield line[t[0]:t[2]]
288+
289+ def sub(self, regex, sub, line, count=0):
290+ """
291+ This is similar to re.sub:
292+
293+ It matches a regex that it is followed by a delimiter,
294+ replacing occurrences only if all delimiters are paired.
295+
296+ if r'\1' is used, it works just like re: it places there the
297+ matched paired data with the delimiter stripped.
298+
299+ If count is different than zero, it will replace at most count
300+ items.
301+ """
302+ out = ""
303+
304+ cur_pos = 0
305+ n = 0
306+
307+ found = False
308+ for start, end, pos in self._search(regex, line):
309+ out += line[cur_pos:start]
310+
311+ # Value, ignoring start/end delimiters
312+ value = line[end:pos - 1]
313+
314+ # replaces \1 at the sub string, if \1 is used there
315+ new_sub = sub
316+ new_sub = new_sub.replace(r'\1', value)
317+
318+ out += new_sub
319+
320+ # Drop end ';' if any
321+ if line[pos] == ';':
322+ pos += 1
323+
324+ cur_pos = pos
325+ n += 1
326+
327+ if count and count >= n:
328+ break
329+
330+ # Append the remaining string
331+ l = len(line)
332+ out += line[cur_pos:l]
333+
334+ return out
335+
170336 #
171337 # Regular expressions used to parse kernel-doc markups at KernelDoc class.
172338 #
@@ -738,22 +904,49 @@ class KernelDoc: · 565 unmodified lines
738904 (Re(r'\s*____cacheline_aligned_in_smp', re.S), ' '),
739905 (Re(r'\s*____cacheline_aligned', re.S), ' '),
740906
741− # Unwrap struct_group() based on this definition:
907+ # Unwrap struct_group macros based on this definition:
742908 # __struct_group(TAG, NAME, ATTRS, MEMBERS...)
743909 # which has variants like: struct_group(NAME, MEMBERS...)
910+ # Only MEMBERS arguments require documentation.
911+ #
912+ # Parsing them happens on two steps:
913+ #
914+ # 1. drop struct group arguments that aren't at MEMBERS,
915+ # storing them as STRUCT_GROUP(MEMBERS)
916+ #
917+ # 2. remove STRUCT_GROUP() ancillary macro.
918+ #
919+ # The original logic used to remove STRUCT_GROUP() using an
920+ # advanced regex:
921+ #
922+ # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*;
923+ #
924+ # with two patterns that are incompatible with
925+ # Python re module, as it has:
926+ #
927+ # - a recursive pattern: (?1)
928+ # - an atomic grouping: (?>...)
929+ #
930+ # I tried a simpler version: but it didn't work either:
931+ # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*;
932+ #
933+ # As it doesn't properly match the end parenthesis on some cases.
934+ #
935+ # So, a better solution was crafted: there's now a NestedMatch
936+ # class that ensures that delimiters after a search are properly
937+ # matched. So, the implementation to drop STRUCT_GROUP() will be
938+ # handled in separate.
744939
745940 (Re(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('),
746941 (Re(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('),
747942 (Re(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('),
748943 (Re(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('),
749944
750− # This is incompatible with Python re, as it uses:
751− # recursive patterns ((?1)) and atomic grouping ((?>...)):
752− # '\bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*;'
753− # Let's see if this works instead:
754− (Re(r'\bSTRUCT_GROUP\(([^\)]+)\)[^;]*;', re.S), r'\1'),
755−
756945 # Replace macros
946+ #
947+ # TODO: it is better to also move those to the NestedMatch logic,
948+ # to ensure that parenthesis will be properly matched.
949+
757950 (Re(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'),
758951 (Re(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'),
759952 (Re(r'DECLARE_BITMAP\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'),
@@ -765,9 +958,22 @@ class KernelDoc: · 5 unmodified lines
765958 (Re(r'DEFINE_DMA_UNMAP_LEN\s*\(' + args_pattern + r'\)', re.S), r'__u32 \1'),
766959 ]
767960
961+ # Regexes here are guaranteed to have the end limiter matching
962+ # the start delimiter. Yet, right now, only one replace group
963+ # is allowed.
964+
965+ sub_nested_prefixes = [
966+ (re.compile(r'\bSTRUCT_GROUP\('), r'\1'),
967+ ]
968+
768969 for search, sub in sub_prefixes:
769970 members = search.sub(sub, members)
770971
972+ nested = NestedMatch()
973+
974+ for search, sub in sub_nested_prefixes:
975+ members = nested.sub(search, sub, members)
976+
771977 # Keeps the original declaration as-is
772978 declaration = members
773979