1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# Copyright Thomas Gleixner <[email protected]> 4 5from argparse import ArgumentParser 6from ply import lex, yacc 7import locale 8import traceback 9import sys 10import git 11import re 12import os 13 14class ParserException(Exception): 15 def __init__(self, tok, txt): 16 self.tok = tok 17 self.txt = txt 18 19class SPDXException(Exception): 20 def __init__(self, el, txt): 21 self.el = el 22 self.txt = txt 23 24class SPDXdata(object): 25 def __init__(self): 26 self.license_files = 0 27 self.exception_files = 0 28 self.licenses = [ ] 29 self.exceptions = { } 30 31class dirinfo(object): 32 def __init__(self): 33 self.missing = 0 34 self.total = 0 35 36 def update(self, miss): 37 self.total += 1 38 self.missing += miss 39 40# Read the spdx data from the LICENSES directory 41def read_spdxdata(repo): 42 43 # The subdirectories of LICENSES in the kernel source 44 # Note: exceptions needs to be parsed as last directory. 45 license_dirs = [ "preferred", "dual", "deprecated", "exceptions" ] 46 lictree = repo.head.commit.tree['LICENSES'] 47 48 spdx = SPDXdata() 49 50 for d in license_dirs: 51 for el in lictree[d].traverse(): 52 if not os.path.isfile(el.path): 53 continue 54 55 exception = None 56 for l in open(el.path, encoding="utf-8").readlines(): 57 if l.startswith('Valid-License-Identifier:'): 58 lid = l.split(':')[1].strip().upper() 59 if lid in spdx.licenses: 60 raise SPDXException(el, 'Duplicate License Identifier: %s' %lid) 61 else: 62 spdx.licenses.append(lid) 63 64 elif l.startswith('SPDX-Exception-Identifier:'): 65 exception = l.split(':')[1].strip().upper() 66 spdx.exceptions[exception] = [] 67 68 elif l.startswith('SPDX-Licenses:'): 69 for lic in l.split(':')[1].upper().strip().replace(' ', '').replace('\t', '').split(','): 70 if not lic in spdx.licenses: 71 raise SPDXException(None, 'Exception %s missing license %s' %(exception, lic)) 72 spdx.exceptions[exception].append(lic) 73 74 elif l.startswith("License-Text:"): 75 if exception: 76 if not len(spdx.exceptions[exception]): 77 raise SPDXException(el, 'Exception %s is missing SPDX-Licenses' %exception) 78 spdx.exception_files += 1 79 else: 80 spdx.license_files += 1 81 break 82 return spdx 83 84class id_parser(object): 85 86 reserved = [ 'AND', 'OR', 'WITH' ] 87 tokens = [ 'LPAR', 'RPAR', 'ID', 'EXC' ] + reserved 88 89 precedence = ( ('nonassoc', 'AND', 'OR'), ) 90 91 t_ignore = ' \t' 92 93 def __init__(self, spdx): 94 self.spdx = spdx 95 self.lasttok = None 96 self.lastid = None 97 self.lexer = lex.lex(module = self, reflags = re.UNICODE) 98 # Initialize the parser. No debug file and no parser rules stored on disk 99 # The rules are small enough to be generated on the fly 100 self.parser = yacc.yacc(module = self, write_tables = False, debug = False) 101 self.lines_checked = 0 102 self.checked = 0 103 self.spdx_valid = 0 104 self.spdx_errors = 0 105 self.spdx_dirs = {} 106 self.curline = 0 107 self.deepest = 0 108 109 # Validate License and Exception IDs 110 def validate(self, tok): 111 id = tok.value.upper() 112 if tok.type == 'ID': 113 if not id in self.spdx.licenses: 114 raise ParserException(tok, 'Invalid License ID') 115 self.lastid = id 116 elif tok.type == 'EXC': 117 if id not in self.spdx.exceptions: 118 raise ParserException(tok, 'Invalid Exception ID') 119 if self.lastid not in self.spdx.exceptions[id]: 120 raise ParserException(tok, 'Exception not valid for license %s' %self.lastid) 121 self.lastid = None 122 elif tok.type != 'WITH': 123 self.lastid = None 124 125 # Lexer functions 126 def t_RPAR(self, tok): 127 r'\)' 128 self.lasttok = tok.type 129 return tok 130 131 def t_LPAR(self, tok): 132 r'\(' 133 self.lasttok = tok.type 134 return tok 135 136 def t_ID(self, tok): 137 r'[A-Za-z.0-9\-+]+' 138 139 if self.lasttok == 'EXC': 140 print(tok) 141 raise ParserException(tok, 'Missing parentheses') 142 143 tok.value = tok.value.strip() 144 val = tok.value.upper() 145 146 if val in self.reserved: 147 tok.type = val 148 elif self.lasttok == 'WITH': 149 tok.type = 'EXC' 150 151 self.lasttok = tok.type 152 self.validate(tok) 153 return tok 154 155 def t_error(self, tok): 156 raise ParserException(tok, 'Invalid token') 157 158 def p_expr(self, p): 159 '''expr : ID 160 | ID WITH EXC 161 | expr AND expr 162 | expr OR expr 163 | LPAR expr RPAR''' 164 pass 165 166 def p_error(self, p): 167 if not p: 168 raise ParserException(None, 'Unfinished license expression') 169 else: 170 raise ParserException(p, 'Syntax error') 171 172 def parse(self, expr): 173 self.lasttok = None 174 self.lastid = None 175 self.parser.parse(expr, lexer = self.lexer) 176 177 def parse_lines(self, fd, maxlines, fname): 178 self.checked += 1 179 self.curline = 0 180 fail = 1 181 try: 182 for line in fd: 183 line = line.decode(locale.getpreferredencoding(False), errors='ignore') 184 self.curline += 1 185 if self.curline > maxlines: 186 break 187 self.lines_checked += 1 188 if line.find("SPDX-License-Identifier:") < 0: 189 continue 190 expr = line.split(':')[1].strip() 191 # Remove trailing comment closure 192 if line.strip().endswith('*/'): 193 expr = expr.rstrip('*/').strip() 194 # Remove trailing xml comment closure 195 if line.strip().endswith('-->'): 196 expr = expr.rstrip('-->').strip() 197 # Special case for SH magic boot code files 198 if line.startswith('LIST \"'): 199 expr = expr.rstrip('\"').strip() 200 self.parse(expr) 201 self.spdx_valid += 1 202 # 203 # Should we check for more SPDX ids in the same file and 204 # complain if there are any? 205 # 206 fail = 0 207 break 208 209 except ParserException as pe: 210 if pe.tok: 211 col = line.find(expr) + pe.tok.lexpos 212 tok = pe.tok.value 213 sys.stdout.write('%s: %d:%d %s: %s\n' %(fname, self.curline, col, pe.txt, tok)) 214 else: 215 sys.stdout.write('%s: %d:0 %s\n' %(fname, self.curline, pe.txt)) 216 self.spdx_errors += 1 217 218 base = os.path.dirname(fname) 219 di = self.spdx_dirs.get(base, dirinfo()) 220 di.update(fail) 221 self.spdx_dirs[base] = di 222 223def scan_git_tree(tree): 224 for el in tree.traverse(): 225 # Exclude stuff which would make pointless noise 226 # FIXME: Put this somewhere more sensible 227 if el.path.startswith("LICENSES"): 228 continue 229 if el.path.find("license-rules.rst") >= 0: 230 continue 231 if not os.path.isfile(el.path): 232 continue 233 with open(el.path, 'rb') as fd: 234 parser.parse_lines(fd, args.maxlines, el.path) 235 236def scan_git_subtree(tree, path): 237 for p in path.strip('/').split('/'): 238 tree = tree[p] 239 scan_git_tree(tree) 240 241if __name__ == '__main__': 242 243 ap = ArgumentParser(description='SPDX expression checker') 244 ap.add_argument('path', nargs='*', help='Check path or file. If not given full git tree scan. For stdin use "-"') 245 ap.add_argument('-m', '--maxlines', type=int, default=15, 246 help='Maximum number of lines to scan in a file. Default 15') 247 ap.add_argument('-v', '--verbose', action='store_true', help='Verbose statistics output') 248 args = ap.parse_args() 249 250 # Sanity check path arguments 251 if '-' in args.path and len(args.path) > 1: 252 sys.stderr.write('stdin input "-" must be the only path argument\n') 253 sys.exit(1) 254 255 try: 256 # Use git to get the valid license expressions 257 repo = git.Repo(os.getcwd()) 258 assert not repo.bare 259 260 # Initialize SPDX data 261 spdx = read_spdxdata(repo) 262 263 # Initialize the parser 264 parser = id_parser(spdx) 265 266 except SPDXException as se: 267 if se.el: 268 sys.stderr.write('%s: %s\n' %(se.el.path, se.txt)) 269 else: 270 sys.stderr.write('%s\n' %se.txt) 271 sys.exit(1) 272 273 except Exception as ex: 274 sys.stderr.write('FAIL: %s\n' %ex) 275 sys.stderr.write('%s\n' %traceback.format_exc()) 276 sys.exit(1) 277 278 try: 279 if len(args.path) and args.path[0] == '-': 280 stdin = os.fdopen(sys.stdin.fileno(), 'rb') 281 parser.parse_lines(stdin, args.maxlines, '-') 282 else: 283 if args.path: 284 for p in args.path: 285 if os.path.isfile(p): 286 parser.parse_lines(open(p, 'rb'), args.maxlines, p) 287 elif os.path.isdir(p): 288 scan_git_subtree(repo.head.reference.commit.tree, p) 289 else: 290 sys.stderr.write('path %s does not exist\n' %p) 291 sys.exit(1) 292 else: 293 # Full git tree scan 294 scan_git_tree(repo.head.commit.tree) 295 296 if args.verbose: 297 sys.stderr.write('\n') 298 sys.stderr.write('License files: %12d\n' %spdx.license_files) 299 sys.stderr.write('Exception files: %12d\n' %spdx.exception_files) 300 sys.stderr.write('License IDs %12d\n' %len(spdx.licenses)) 301 sys.stderr.write('Exception IDs %12d\n' %len(spdx.exceptions)) 302 sys.stderr.write('\n') 303 sys.stderr.write('Files checked: %12d\n' %parser.checked) 304 sys.stderr.write('Lines checked: %12d\n' %parser.lines_checked) 305 if parser.checked: 306 pc = int(100 * parser.spdx_valid / parser.checked) 307 sys.stderr.write('Files with SPDX: %12d %3d%%\n' %(parser.spdx_valid, pc)) 308 sys.stderr.write('Files with errors: %12d\n' %parser.spdx_errors) 309 ndirs = len(parser.spdx_dirs) 310 dirsok = 0 311 if ndirs: 312 sys.stderr.write('\n') 313 sys.stderr.write('Directories accounted: %8d\n' %ndirs) 314 for di in parser.spdx_dirs.values(): 315 if not di.missing: 316 dirsok += 1 317 pc = int(100 * dirsok / ndirs) 318 sys.stderr.write('Directories complete: %8d %3d%%\n' %(dirsok, pc)) 319 320 sys.exit(0) 321 322 except Exception as ex: 323 sys.stderr.write('FAIL: %s\n' %ex) 324 sys.stderr.write('%s\n' %traceback.format_exc()) 325 sys.exit(1) 326