Handing nested parenthesis with regular expressions is not an easy task. It is even harder with Python's re module, as it has a limited subset of regular expressions, missing more advanced features. We might use instead Python regex module, but still the regular expressions are very hard to understand. So, instead, add a logic to properly match delimiters.
Unified · Split
| @@ -167,6 +167,172 @@ class Re: · 166 unmodified lines | |||
| 167 | def group(self, num): | 167 | def group(self, num): |
| 168 | return self.last_match.group(num) | 168 | return self.last_match.group(num) |
| 169 | 169 | ||
| 170 | class NestedMatch: | ||
| 171 | """ | ||
| 172 | Finding nested delimiters is hard with regular expressions. It is | ||
| 173 | even harder on Python with its normal re module, as there are several | ||
| 174 | advanced regular expressions that are missing. | ||
| 175 | |||
| 176 | This is the case of this pattern: | ||
| 177 | |||
| 178 | '\\bSTRUCT_GROUP(\\(((?:(?>[^)(]+)|(?1))*)\\))[^;]*;' | ||
| 179 | |||
| 180 | which is used to properly match open/close parenthesis of the | ||
| 181 | string search STRUCT_GROUP(), | ||
| 182 | |||
| 183 | Add a class that counts pairs of delimiters, using it to match and | ||
| 184 | replace nested expressions. | ||
| 185 | |||
| 186 | The original approach was suggested by: | ||
| 187 | https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex | ||
| 188 | |||
| 189 | Although I re-implemented it to make it more generic and match 3 types | ||
| 190 | of delimiters. The logic checks if delimiters are paired. If not, it | ||
| 191 | will ignore the search string. | ||
| 192 | """ | ||
| 193 | |||
| 194 | # TODO: | ||
| 195 | # Right now, regular expressions to match it are defined only up to | ||
| 196 | # the start delimiter, e.g.: | ||
| 197 | # | ||
| 198 | # \bSTRUCT_GROUP\( | ||
| 199 | # | ||
| 200 | # is similar to: STRUCT_GROUP\((.*)\) | ||
| 201 | # except that the content inside the match group is delimiter's aligned. | ||
| 202 | # | ||
| 203 | # The content inside parenthesis are converted into a single replace | ||
| 204 | # group (e.g. r`\1'). | ||
| 205 | # | ||
| 206 | # It would be nice to change such definition to support multiple | ||
| 207 | # match groups, allowing a regex equivalent to. | ||
| 208 | # | ||
| 209 | # FOO\((.*), (.*), (.*)\) | ||
| 210 | # | ||
| 211 | # it is probably easier to define it not as a regular expression, but | ||
| 212 | # with some lexical definition like: | ||
| 213 | # | ||
| 214 | # FOO(arg1, arg2, arg3) | ||
| 215 | |||
| 216 | |||
| 217 | DELIMITER_PAIRS = { | ||
| 218 | '{': '}', | ||
| 219 | '(': ')', | ||
| 220 | '[': ']', | ||
| 221 | } | ||
| 222 | |||
| 223 | RE_DELIM = re.compile(r'[\{\}\[\]\(\)]') | ||
| 224 | |||
| 225 | def _search(self, regex, line): | ||
| 226 | """ | ||
| 227 | Finds paired blocks for a regex that ends with a delimiter. | ||
| 228 | |||
| 229 | The suggestion of using finditer to match pairs came from: | ||
| 230 | https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex | ||
| 231 | but I ended using a different implementation to align all three types | ||
| 232 | of delimiters and seek for an initial regular expression. | ||
| 233 | |||
| 234 | The algorithm seeks for open/close paired delimiters and place them | ||
| 235 | into a stack, yielding a start/stop position of each match when the | ||
| 236 | stack is zeroed. | ||
| 237 | |||
| 238 | The algorithm shoud work fine for properly paired lines, but will | ||
| 239 | silently ignore end delimiters that preceeds an start delimiter. | ||
| 240 | This should be OK for kernel-doc parser, as unaligned delimiters | ||
| 241 | would cause compilation errors. So, we don't need to rise exceptions | ||
| 242 | to cover such issues. | ||
| 243 | """ | ||
| 244 | |||
| 245 | stack = [] | ||
| 246 | |||
| 247 | for match_re in regex.finditer(line): | ||
| 248 | start = match_re.start() | ||
| 249 | offset = match_re.end() | ||
| 250 | |||
| 251 | d = line[offset -1] | ||
| 252 | if d not in self.DELIMITER_PAIRS: | ||
| 253 | continue | ||
| 254 | |||
| 255 | end = self.DELIMITER_PAIRS[d] | ||
| 256 | stack.append(end) | ||
| 257 | |||
| 258 | for match in self.RE_DELIM.finditer(line[offset:]): | ||
| 259 | pos = match.start() + offset | ||
| 260 | |||
| 261 | d = line[pos] | ||
| 262 | |||
| 263 | if d in self.DELIMITER_PAIRS: | ||
| 264 | end = self.DELIMITER_PAIRS[d] | ||
| 265 | |||
| 266 | stack.append(end) | ||
| 267 | continue | ||
| 268 | |||
| 269 | # Does the end delimiter match what it is expected? | ||
| 270 | if stack and d == stack[-1]: | ||
| 271 | stack.pop() | ||
| 272 | |||
| 273 | if not stack: | ||
| 274 | yield start, offset, pos + 1 | ||
| 275 | break | ||
| 276 | |||
| 277 | def search(self, regex, line): | ||
| 278 | """ | ||
| 279 | This is similar to re.search: | ||
| 280 | |||
| 281 | It matches a regex that it is followed by a delimiter, | ||
| 282 | returning occurrences only if all delimiters are paired. | ||
| 283 | """ | ||
| 284 | |||
| 285 | for t in self._search(regex, line): | ||
| 286 | |||
| 287 | yield line[t[0]:t[2]] | ||
| 288 | |||
| 289 | def sub(self, regex, sub, line, count=0): | ||
| 290 | """ | ||
| 291 | This is similar to re.sub: | ||
| 292 | |||
| 293 | It matches a regex that it is followed by a delimiter, | ||
| 294 | replacing occurrences only if all delimiters are paired. | ||
| 295 | |||
| 296 | if r'\1' is used, it works just like re: it places there the | ||
| 297 | matched paired data with the delimiter stripped. | ||
| 298 | |||
| 299 | If count is different than zero, it will replace at most count | ||
| 300 | items. | ||
| 301 | """ | ||
| 302 | out = "" | ||
| 303 | |||
| 304 | cur_pos = 0 | ||
| 305 | n = 0 | ||
| 306 | |||
| 307 | found = False | ||
| 308 | for start, end, pos in self._search(regex, line): | ||
| 309 | out += line[cur_pos:start] | ||
| 310 | |||
| 311 | # Value, ignoring start/end delimiters | ||
| 312 | value = line[end:pos - 1] | ||
| 313 | |||
| 314 | # replaces \1 at the sub string, if \1 is used there | ||
| 315 | new_sub = sub | ||
| 316 | new_sub = new_sub.replace(r'\1', value) | ||
| 317 | |||
| 318 | out += new_sub | ||
| 319 | |||
| 320 | # Drop end ';' if any | ||
| 321 | if line[pos] == ';': | ||
| 322 | pos += 1 | ||
| 323 | |||
| 324 | cur_pos = pos | ||
| 325 | n += 1 | ||
| 326 | |||
| 327 | if count and count >= n: | ||
| 328 | break | ||
| 329 | |||
| 330 | # Append the remaining string | ||
| 331 | l = len(line) | ||
| 332 | out += line[cur_pos:l] | ||
| 333 | |||
| 334 | return out | ||
| 335 | |||
| 170 | # | 336 | # |
| 171 | # Regular expressions used to parse kernel-doc markups at KernelDoc class. | 337 | # Regular expressions used to parse kernel-doc markups at KernelDoc class. |
| 172 | # | 338 | # |
| @@ -738,22 +904,49 @@ class KernelDoc: · 565 unmodified lines | |||
| 738 | (Re(r'\s*____cacheline_aligned_in_smp', re.S), ' '), | 904 | (Re(r'\s*____cacheline_aligned_in_smp', re.S), ' '), |
| 739 | (Re(r'\s*____cacheline_aligned', re.S), ' '), | 905 | (Re(r'\s*____cacheline_aligned', re.S), ' '), |
| 740 | 906 | ||
| 741 | # Unwrap struct_group() based on this definition: | 907 | # Unwrap struct_group macros based on this definition: |
| 742 | # __struct_group(TAG, NAME, ATTRS, MEMBERS...) | 908 | # __struct_group(TAG, NAME, ATTRS, MEMBERS...) |
| 743 | # which has variants like: struct_group(NAME, MEMBERS...) | 909 | # which has variants like: struct_group(NAME, MEMBERS...) |
| 910 | # Only MEMBERS arguments require documentation. | ||
| 911 | # | ||
| 912 | # Parsing them happens on two steps: | ||
| 913 | # | ||
| 914 | # 1. drop struct group arguments that aren't at MEMBERS, | ||
| 915 | # storing them as STRUCT_GROUP(MEMBERS) | ||
| 916 | # | ||
| 917 | # 2. remove STRUCT_GROUP() ancillary macro. | ||
| 918 | # | ||
| 919 | # The original logic used to remove STRUCT_GROUP() using an | ||
| 920 | # advanced regex: | ||
| 921 | # | ||
| 922 | # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; | ||
| 923 | # | ||
| 924 | # with two patterns that are incompatible with | ||
| 925 | # Python re module, as it has: | ||
| 926 | # | ||
| 927 | # - a recursive pattern: (?1) | ||
| 928 | # - an atomic grouping: (?>...) | ||
| 929 | # | ||
| 930 | # I tried a simpler version: but it didn't work either: | ||
| 931 | # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; | ||
| 932 | # | ||
| 933 | # As it doesn't properly match the end parenthesis on some cases. | ||
| 934 | # | ||
| 935 | # So, a better solution was crafted: there's now a NestedMatch | ||
| 936 | # class that ensures that delimiters after a search are properly | ||
| 937 | # matched. So, the implementation to drop STRUCT_GROUP() will be | ||
| 938 | # handled in separate. | ||
| 744 | 939 | ||
| 745 | (Re(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), | 940 | (Re(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), |
| 746 | (Re(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), | 941 | (Re(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), |
| 747 | (Re(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), | 942 | (Re(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), |
| 748 | (Re(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), | 943 | (Re(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), |
| 749 | 944 | ||
| 750 | # This is incompatible with Python re, as it uses: | ||
| 751 | # recursive patterns ((?1)) and atomic grouping ((?>...)): | ||
| 752 | # '\bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*;' | ||
| 753 | # Let's see if this works instead: | ||
| 754 | (Re(r'\bSTRUCT_GROUP\(([^\)]+)\)[^;]*;', re.S), r'\1'), | ||
| 755 | |||
| 756 | # Replace macros | 945 | # Replace macros |
| 946 | # | ||
| 947 | # TODO: it is better to also move those to the NestedMatch logic, | ||
| 948 | # to ensure that parenthesis will be properly matched. | ||
| 949 | |||
| 757 | (Re(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), | 950 | (Re(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), |
| 758 | (Re(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), | 951 | (Re(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), |
| 759 | (Re(r'DECLARE_BITMAP\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), | 952 | (Re(r'DECLARE_BITMAP\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), |
| @@ -765,9 +958,22 @@ class KernelDoc: · 5 unmodified lines | |||
| 765 | (Re(r'DEFINE_DMA_UNMAP_LEN\s*\(' + args_pattern + r'\)', re.S), r'__u32 \1'), | 958 | (Re(r'DEFINE_DMA_UNMAP_LEN\s*\(' + args_pattern + r'\)', re.S), r'__u32 \1'), |
| 766 | ] | 959 | ] |
| 767 | 960 | ||
| 961 | # Regexes here are guaranteed to have the end limiter matching | ||
| 962 | # the start delimiter. Yet, right now, only one replace group | ||
| 963 | # is allowed. | ||
| 964 | |||
| 965 | sub_nested_prefixes = [ | ||
| 966 | (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), | ||
| 967 | ] | ||
| 968 | |||
| 768 | for search, sub in sub_prefixes: | 969 | for search, sub in sub_prefixes: |
| 769 | members = search.sub(sub, members) | 970 | members = search.sub(sub, members) |
| 770 | 971 | ||
| 972 | nested = NestedMatch() | ||
| 973 | |||
| 974 | for search, sub in sub_nested_prefixes: | ||
| 975 | members = nested.sub(search, sub, members) | ||
| 976 | |||
| 771 | # Keeps the original declaration as-is | 977 | # Keeps the original declaration as-is |
| 772 | declaration = members | 978 | declaration = members |
| 773 | 979 | ||