Pull documentation updates from Jonathan Corbet: "There's been a fair amount of work in the docs tree this time around, including: - Extensive RST conversions and organizational work in the memory-management docs thanks to Mike Rapoport. - An update of Documentation/features from Andrea Parri and a script to keep it updated. - Various LICENSES updates from Thomas, along with a script to check SPDX tags. - Work to fix dangling references to documentation files; this involved a fair number of one-liner comment changes outside of Documentation/ ... and the usual list of documentation improvements, typo fixes, etc" * tag 'docs-4.18' of git://git.lwn.net/linux: (103 commits) Documentation: document hung_task_panic kernel parameter docs/admin-guide/mm: add high level concepts overview docs/vm: move ksm and transhuge from "user" to "internals" section. docs: Use the kerneldoc comments for memalloc_no*() doc: document scope NOFS, NOIO APIs docs: update kernel versions and dates in tables docs/vm: transhuge: split userspace bits to admin-guide/mm/transhuge docs/vm: transhuge: minor updates docs/vm: transhuge: change sections order Documentation: arm: clean up Marvell Berlin family info Documentation: gpio: driver: Fix a typo and some odd grammar docs: ranoops.rst: fix location of ramoops.txt scripts/documentation-file-ref-check: rewrite it in perl with auto-fix mode docs: uio-howto.rst: use a code block to solve a warning mm, THP, doc: Add document for thp_swpout/thp_swpout_fallback w1: w1_io.c: fix a kernel-doc warning Documentation/process/posting: wrap text at 80 cols docs: admin-guide: add cgroup-v2 documentation Revert "Documentation/features/vm: Remove arch support status file for 'pte_special'" Documentation: refcount-vs-atomic: Update reference to LKMM doc. ...
Showing the changes to scripts (2 of 166 files). Show all
Unified · Split
| @@ -1,15 +1,116 @@ | |||
| 1 | #!/bin/sh | 1 | #!/usr/bin/env perl |
| 2 | # SPDX-License-Identifier: GPL-2.0 | ||
| 3 | # | ||
| 2 | # Treewide grep for references to files under Documentation, and report | 4 | # Treewide grep for references to files under Documentation, and report |
| 3 | # non-existing files in stderr. | 5 | # non-existing files in stderr. |
| 4 | 6 | ||
| 5 | for f in $(git ls-files); do | 7 | use warnings; |
| 6 | for ref in $(grep -ho "Documentation/[A-Za-z0-9_.,~/*+-]*" "$f"); do | 8 | use strict; |
| 7 | # presume trailing . and , are not part of the name | 9 | use Getopt::Long qw(:config no_auto_abbrev); |
| 8 | ref=${ref%%[.,]} | 10 | |
| 9 | 11 | my $scriptname = $0; | |
| 10 | # use ls to handle wildcards | 12 | $scriptname =~ s,.*/([^/]+/),$1,; |
| 11 | if ! ls $ref >/dev/null 2>&1; then | 13 | |
| 12 | echo "$f: $ref" >&2 | 14 | # Parse arguments |
| 13 | fi | 15 | my $help = 0; |
| 14 | done | 16 | my $fix = 0; |
| 15 | done | 17 | |
| 18 | GetOptions( | ||
| 19 | 'fix' => \$fix, | ||
| 20 | 'h|help|usage' => \$help, | ||
| 21 | ); | ||
| 22 | |||
| 23 | if ($help != 0) { | ||
| 24 | print "$scriptname [--help] [--fix-rst]\n"; | ||
| 25 | exit -1; | ||
| 26 | } | ||
| 27 | |||
| 28 | # Step 1: find broken references | ||
| 29 | print "Finding broken references. This may take a while... " if ($fix); | ||
| 30 | |||
| 31 | my %broken_ref; | ||
| 32 | |||
| 33 | open IN, "git grep 'Documentation/'|" | ||
| 34 | or die "Failed to run git grep"; | ||
| 35 | while (<IN>) { | ||
| 36 | next if (!m/^([^:]+):(.*)/); | ||
| 37 | |||
| 38 | my $f = $1; | ||
| 39 | my $ln = $2; | ||
| 40 | |||
| 41 | # Makefiles contain nasty expressions to parse docs | ||
| 42 | next if ($f =~ m/Makefile/); | ||
| 43 | # Skip this script | ||
| 44 | next if ($f eq $scriptname); | ||
| 45 | |||
| 46 | if ($ln =~ m,\b(\S*)(Documentation/[A-Za-z0-9\_\.\,\~/\*+-]*),) { | ||
| 47 | my $prefix = $1; | ||
| 48 | my $ref = $2; | ||
| 49 | my $base = $2; | ||
| 50 | |||
| 51 | $ref =~ s/[\,\.]+$//; | ||
| 52 | |||
| 53 | my $fulref = "$prefix$ref"; | ||
| 54 | |||
| 55 | $fulref =~ s/^(\<file|ref)://; | ||
| 56 | $fulref =~ s/^[\'\`]+//; | ||
| 57 | $fulref =~ s,^\$\(.*\)/,,; | ||
| 58 | $base =~ s,.*/,,; | ||
| 59 | |||
| 60 | # Remove URL false-positives | ||
| 61 | next if ($fulref =~ m/^http/); | ||
| 62 | |||
| 63 | # Check if exists, evaluating wildcards | ||
| 64 | next if (grep -e, glob("$ref $fulref")); | ||
| 65 | |||
| 66 | if ($fix) { | ||
| 67 | if (!($ref =~ m/(devicetree|scripts|Kconfig|Kbuild)/)) { | ||
| 68 | $broken_ref{$ref}++; | ||
| 69 | } | ||
| 70 | } else { | ||
| 71 | print STDERR "$f: $fulref\n"; | ||
| 72 | } | ||
| 73 | } | ||
| 74 | } | ||
| 75 | |||
| 76 | exit 0 if (!$fix); | ||
| 77 | |||
| 78 | # Step 2: Seek for file name alternatives | ||
| 79 | print "Auto-fixing broken references. Please double-check the results\n"; | ||
| 80 | |||
| 81 | foreach my $ref (keys %broken_ref) { | ||
| 82 | my $new =$ref; | ||
| 83 | |||
| 84 | # get just the basename | ||
| 85 | $new =~ s,.*/,,; | ||
| 86 | |||
| 87 | # Seek for the same name on another place, as it may have been moved | ||
| 88 | my $f=""; | ||
| 89 | |||
| 90 | $f = qx(find . -iname $new) if ($new); | ||
| 91 | |||
| 92 | # usual reason for breakage: file renamed to .rst | ||
| 93 | if (!$f) { | ||
| 94 | $new =~ s/\.txt$/.rst/; | ||
| 95 | $f=qx(find . -iname $new) if ($new); | ||
| 96 | } | ||
| 97 | |||
| 98 | my @find = split /\s+/, $f; | ||
| 99 | |||
| 100 | if (!$f) { | ||
| 101 | print STDERR "ERROR: Didn't find a replacement for $ref\n"; | ||
| 102 | } elsif (scalar(@find) > 1) { | ||
| 103 | print STDERR "WARNING: Won't auto-replace, as found multiple files close to $ref:\n"; | ||
| 104 | foreach my $j (@find) { | ||
| 105 | $j =~ s,^./,,; | ||
| 106 | print STDERR " $j\n"; | ||
| 107 | } | ||
| 108 | } else { | ||
| 109 | $f = $find[0]; | ||
| 110 | $f =~ s,^./,,; | ||
| 111 | print "INFO: Replacing $ref to $f\n"; | ||
| 112 | foreach my $j (qx(git grep -l $ref)) { | ||
| 113 | qx(sed "s\@$ref\@$f\@g" -i $j); | ||
| 114 | } | ||
| 115 | } | ||
| 116 | } | ||
| @@ -0,0 +1,284 @@ | |||
| 1 | #!/usr/bin/env python | ||
| 2 | # SPDX-License-Identifier: GPL-2.0 | ||
| 3 | # Copyright Thomas Gleixner <tglx@linutronix.de> | ||
| 4 | |||
| 5 | from argparse import ArgumentParser | ||
| 6 | from ply import lex, yacc | ||
| 7 | import traceback | ||
| 8 | import sys | ||
| 9 | import git | ||
| 10 | import re | ||
| 11 | import os | ||
| 12 | |||
| 13 | class ParserException(Exception): | ||
| 14 | def __init__(self, tok, txt): | ||
| 15 | self.tok = tok | ||
| 16 | self.txt = txt | ||
| 17 | |||
| 18 | class SPDXException(Exception): | ||
| 19 | def __init__(self, el, txt): | ||
| 20 | self.el = el | ||
| 21 | self.txt = txt | ||
| 22 | |||
| 23 | class SPDXdata(object): | ||
| 24 | def __init__(self): | ||
| 25 | self.license_files = 0 | ||
| 26 | self.exception_files = 0 | ||
| 27 | self.licenses = [ ] | ||
| 28 | self.exceptions = { } | ||
| 29 | |||
| 30 | # Read the spdx data from the LICENSES directory | ||
| 31 | def read_spdxdata(repo): | ||
| 32 | |||
| 33 | # The subdirectories of LICENSES in the kernel source | ||
| 34 | license_dirs = [ "preferred", "other", "exceptions" ] | ||
| 35 | lictree = repo.heads.master.commit.tree['LICENSES'] | ||
| 36 | |||
| 37 | spdx = SPDXdata() | ||
| 38 | |||
| 39 | for d in license_dirs: | ||
| 40 | for el in lictree[d].traverse(): | ||
| 41 | if not os.path.isfile(el.path): | ||
| 42 | continue | ||
| 43 | |||
| 44 | exception = None | ||
| 45 | for l in open(el.path).readlines(): | ||
| 46 | if l.startswith('Valid-License-Identifier:'): | ||
| 47 | lid = l.split(':')[1].strip().upper() | ||
| 48 | if lid in spdx.licenses: | ||
| 49 | raise SPDXException(el, 'Duplicate License Identifier: %s' %lid) | ||
| 50 | else: | ||
| 51 | spdx.licenses.append(lid) | ||
| 52 | |||
| 53 | elif l.startswith('SPDX-Exception-Identifier:'): | ||
| 54 | exception = l.split(':')[1].strip().upper() | ||
| 55 | spdx.exceptions[exception] = [] | ||
| 56 | |||
| 57 | elif l.startswith('SPDX-Licenses:'): | ||
| 58 | for lic in l.split(':')[1].upper().strip().replace(' ', '').replace('\t', '').split(','): | ||
| 59 | if not lic in spdx.licenses: | ||
| 60 | raise SPDXException(None, 'Exception %s missing license %s' %(ex, lic)) | ||
| 61 | spdx.exceptions[exception].append(lic) | ||
| 62 | |||
| 63 | elif l.startswith("License-Text:"): | ||
| 64 | if exception: | ||
| 65 | if not len(spdx.exceptions[exception]): | ||
| 66 | raise SPDXException(el, 'Exception %s is missing SPDX-Licenses' %excid) | ||
| 67 | spdx.exception_files += 1 | ||
| 68 | else: | ||
| 69 | spdx.license_files += 1 | ||
| 70 | break | ||
| 71 | return spdx | ||
| 72 | |||
| 73 | class id_parser(object): | ||
| 74 | |||
| 75 | reserved = [ 'AND', 'OR', 'WITH' ] | ||
| 76 | tokens = [ 'LPAR', 'RPAR', 'ID', 'EXC' ] + reserved | ||
| 77 | |||
| 78 | precedence = ( ('nonassoc', 'AND', 'OR'), ) | ||
| 79 | |||
| 80 | t_ignore = ' \t' | ||
| 81 | |||
| 82 | def __init__(self, spdx): | ||
| 83 | self.spdx = spdx | ||
| 84 | self.lasttok = None | ||
| 85 | self.lastid = None | ||
| 86 | self.lexer = lex.lex(module = self, reflags = re.UNICODE) | ||
| 87 | # Initialize the parser. No debug file and no parser rules stored on disk | ||
| 88 | # The rules are small enough to be generated on the fly | ||
| 89 | self.parser = yacc.yacc(module = self, write_tables = False, debug = False) | ||
| 90 | self.lines_checked = 0 | ||
| 91 | self.checked = 0 | ||
| 92 | self.spdx_valid = 0 | ||
| 93 | self.spdx_errors = 0 | ||
| 94 | self.curline = 0 | ||
| 95 | self.deepest = 0 | ||
| 96 | |||
| 97 | # Validate License and Exception IDs | ||
| 98 | def validate(self, tok): | ||
| 99 | id = tok.value.upper() | ||
| 100 | if tok.type == 'ID': | ||
| 101 | if not id in self.spdx.licenses: | ||
| 102 | raise ParserException(tok, 'Invalid License ID') | ||
| 103 | self.lastid = id | ||
| 104 | elif tok.type == 'EXC': | ||
| 105 | if not self.spdx.exceptions.has_key(id): | ||
| 106 | raise ParserException(tok, 'Invalid Exception ID') | ||
| 107 | if self.lastid not in self.spdx.exceptions[id]: | ||
| 108 | raise ParserException(tok, 'Exception not valid for license %s' %self.lastid) | ||
| 109 | self.lastid = None | ||
| 110 | elif tok.type != 'WITH': | ||
| 111 | self.lastid = None | ||
| 112 | |||
| 113 | # Lexer functions | ||
| 114 | def t_RPAR(self, tok): | ||
| 115 | r'\)' | ||
| 116 | self.lasttok = tok.type | ||
| 117 | return tok | ||
| 118 | |||
| 119 | def t_LPAR(self, tok): | ||
| 120 | r'\(' | ||
| 121 | self.lasttok = tok.type | ||
| 122 | return tok | ||
| 123 | |||
| 124 | def t_ID(self, tok): | ||
| 125 | r'[A-Za-z.0-9\-+]+' | ||
| 126 | |||
| 127 | if self.lasttok == 'EXC': | ||
| 128 | print(tok) | ||
| 129 | raise ParserException(tok, 'Missing parentheses') | ||
| 130 | |||
| 131 | tok.value = tok.value.strip() | ||
| 132 | val = tok.value.upper() | ||
| 133 | |||
| 134 | if val in self.reserved: | ||
| 135 | tok.type = val | ||
| 136 | elif self.lasttok == 'WITH': | ||
| 137 | tok.type = 'EXC' | ||
| 138 | |||
| 139 | self.lasttok = tok.type | ||
| 140 | self.validate(tok) | ||
| 141 | return tok | ||
| 142 | |||
| 143 | def t_error(self, tok): | ||
| 144 | raise ParserException(tok, 'Invalid token') | ||
| 145 | |||
| 146 | def p_expr(self, p): | ||
| 147 | '''expr : ID | ||
| 148 | | ID WITH EXC | ||
| 149 | | expr AND expr | ||
| 150 | | expr OR expr | ||
| 151 | | LPAR expr RPAR''' | ||
| 152 | pass | ||
| 153 | |||
| 154 | def p_error(self, p): | ||
| 155 | if not p: | ||
| 156 | raise ParserException(None, 'Unfinished license expression') | ||
| 157 | else: | ||
| 158 | raise ParserException(p, 'Syntax error') | ||
| 159 | |||
| 160 | def parse(self, expr): | ||
| 161 | self.lasttok = None | ||
| 162 | self.lastid = None | ||
| 163 | self.parser.parse(expr, lexer = self.lexer) | ||
| 164 | |||
| 165 | def parse_lines(self, fd, maxlines, fname): | ||
| 166 | self.checked += 1 | ||
| 167 | self.curline = 0 | ||
| 168 | try: | ||
| 169 | for line in fd: | ||
| 170 | self.curline += 1 | ||
| 171 | if self.curline > maxlines: | ||
| 172 | break | ||
| 173 | self.lines_checked += 1 | ||
| 174 | if line.find("SPDX-License-Identifier:") < 0: | ||
| 175 | continue | ||
| 176 | expr = line.split(':')[1].replace('*/', '').strip() | ||
| 177 | self.parse(expr) | ||
| 178 | self.spdx_valid += 1 | ||
| 179 | # | ||
| 180 | # Should we check for more SPDX ids in the same file and | ||
| 181 | # complain if there are any? | ||
| 182 | # | ||
| 183 | break | ||
| 184 | |||
| 185 | except ParserException as pe: | ||
| 186 | if pe.tok: | ||
| 187 | col = line.find(expr) + pe.tok.lexpos | ||
| 188 | tok = pe.tok.value | ||
| 189 | sys.stdout.write('%s: %d:%d %s: %s\n' %(fname, self.curline, col, pe.txt, tok)) | ||
| 190 | else: | ||
| 191 | sys.stdout.write('%s: %d:0 %s\n' %(fname, self.curline, col, pe.txt)) | ||
| 192 | self.spdx_errors += 1 | ||
| 193 | |||
| 194 | def scan_git_tree(tree): | ||
| 195 | for el in tree.traverse(): | ||
| 196 | # Exclude stuff which would make pointless noise | ||
| 197 | # FIXME: Put this somewhere more sensible | ||
| 198 | if el.path.startswith("LICENSES"): | ||
| 199 | continue | ||
| 200 | if el.path.find("license-rules.rst") >= 0: | ||
| 201 | continue | ||
| 202 | if el.path == 'scripts/checkpatch.pl': | ||
| 203 | continue | ||
| 204 | if not os.path.isfile(el.path): | ||
| 205 | continue | ||
| 206 | parser.parse_lines(open(el.path), args.maxlines, el.path) | ||
| 207 | |||
| 208 | def scan_git_subtree(tree, path): | ||
| 209 | for p in path.strip('/').split('/'): | ||
| 210 | tree = tree[p] | ||
| 211 | scan_git_tree(tree) | ||
| 212 | |||
| 213 | if __name__ == '__main__': | ||
| 214 | |||
| 215 | ap = ArgumentParser(description='SPDX expression checker') | ||
| 216 | ap.add_argument('path', nargs='*', help='Check path or file. If not given full git tree scan. For stdin use "-"') | ||
| 217 | ap.add_argument('-m', '--maxlines', type=int, default=15, | ||
| 218 | help='Maximum number of lines to scan in a file. Default 15') | ||
| 219 | ap.add_argument('-v', '--verbose', action='store_true', help='Verbose statistics output') | ||
| 220 | args = ap.parse_args() | ||
| 221 | |||
| 222 | # Sanity check path arguments | ||
| 223 | if '-' in args.path and len(args.path) > 1: | ||
| 224 | sys.stderr.write('stdin input "-" must be the only path argument\n') | ||
| 225 | sys.exit(1) | ||
| 226 | |||
| 227 | try: | ||
| 228 | # Use git to get the valid license expressions | ||
| 229 | repo = git.Repo(os.getcwd()) | ||
| 230 | assert not repo.bare | ||
| 231 | |||
| 232 | # Initialize SPDX data | ||
| 233 | spdx = read_spdxdata(repo) | ||
| 234 | |||
| 235 | # Initilize the parser | ||
| 236 | parser = id_parser(spdx) | ||
| 237 | |||
| 238 | except SPDXException as se: | ||
| 239 | if se.el: | ||
| 240 | sys.stderr.write('%s: %s\n' %(se.el.path, se.txt)) | ||
| 241 | else: | ||
| 242 | sys.stderr.write('%s\n' %se.txt) | ||
| 243 | sys.exit(1) | ||
| 244 | |||
| 245 | except Exception as ex: | ||
| 246 | sys.stderr.write('FAIL: %s\n' %ex) | ||
| 247 | sys.stderr.write('%s\n' %traceback.format_exc()) | ||
| 248 | sys.exit(1) | ||
| 249 | |||
| 250 | try: | ||
| 251 | if len(args.path) and args.path[0] == '-': | ||
| 252 | parser.parse_lines(sys.stdin, args.maxlines, '-') | ||
| 253 | else: | ||
| 254 | if args.path: | ||
| 255 | for p in args.path: | ||
| 256 | if os.path.isfile(p): | ||
| 257 | parser.parse_lines(open(p), args.maxlines, p) | ||
| 258 | elif os.path.isdir(p): | ||
| 259 | scan_git_subtree(repo.head.reference.commit.tree, p) | ||
| 260 | else: | ||
| 261 | sys.stderr.write('path %s does not exist\n' %p) | ||
| 262 | sys.exit(1) | ||
| 263 | else: | ||
| 264 | # Full git tree scan | ||
| 265 | scan_git_tree(repo.head.commit.tree) | ||
| 266 | |||
| 267 | if args.verbose: | ||
| 268 | sys.stderr.write('\n') | ||
| 269 | sys.stderr.write('License files: %12d\n' %spdx.license_files) | ||
| 270 | sys.stderr.write('Exception files: %12d\n' %spdx.exception_files) | ||
| 271 | sys.stderr.write('License IDs %12d\n' %len(spdx.licenses)) | ||
| 272 | sys.stderr.write('Exception IDs %12d\n' %len(spdx.exceptions)) | ||
| 273 | sys.stderr.write('\n') | ||
| 274 | sys.stderr.write('Files checked: %12d\n' %parser.checked) | ||
| 275 | sys.stderr.write('Lines checked: %12d\n' %parser.lines_checked) | ||
| 276 | sys.stderr.write('Files with SPDX: %12d\n' %parser.spdx_valid) | ||
| 277 | sys.stderr.write('Files with errors: %12d\n' %parser.spdx_errors) | ||
| 278 | |||
| 279 | sys.exit(0) | ||
| 280 | |||
| 281 | except Exception as ex: | ||
| 282 | sys.stderr.write('FAIL: %s\n' %ex) | ||
| 283 | sys.stderr.write('%s\n' %traceback.format_exc()) | ||
| 284 | sys.exit(1) | ||