Commit 0e7f0306 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Greg Kroah-Hartman
Browse files

scripts/spdxcheck: Add [sub]directory statistics 



Add functionality to display [sub]directory statistics. This is enabled by
adding '-d' to the command line. The optional -D parameter allows to limit
the directory depth. If supplied the subdirectories are accumulated

# scripts/spdxcheck.py -d kernel/
Incomplete directories: SPDX in Files
    ./kernel                         :   111 of   114   97%
    ./kernel/bpf                     :    43 of    45   95%
    ./kernel/bpf/preload             :     4 of     5   80%
    ./kernel/bpf/preload/iterators   :     4 of     5   80%
    ./kernel/cgroup                  :    10 of    13   76%
    ./kernel/configs                 :     0 of     9    0%
    ./kernel/debug                   :     3 of     4   75%
    ./kernel/debug/kdb               :     1 of    11    9%
    ./kernel/locking                 :    29 of    32   90%
    ./kernel/sched                   :    38 of    39   97%

The result can be accumulated by restricting the depth via the new command
line option '-d $DEPTH':

# scripts/spdxcheck.py -d -D1
Incomplete directories: SPDX in Files
    ./                               :     6 of    13   46%
    ./Documentation                  :  4096 of  8451   48%
    ./arch                           : 13476 of 16402   82%
    ./block                          :   100 of   101   99%
    ./certs                          :    11 of    14   78%
    ./crypto                         :   145 of   176   82%
    ./drivers                        : 24682 of 30745   80%
    ./fs                             :  1876 of  2110   88%
    ./include                        :  5175 of  5757   89%
    ./ipc                            :    12 of    13   92%
    ./kernel                         :   493 of   527   93%
    ./lib                            :   393 of   524   75%
    ./mm                             :   151 of   159   94%
    ./net                            :  1713 of  1900   90%
    ./samples                        :   211 of   273   77%
    ./scripts                        :   341 of   435   78%
    ./security                       :   241 of   250   96%
    ./sound                          :  2438 of  2503   97%
    ./tools                          :  3810 of  5462   69%
    ./usr                            :     9 of    10   90%

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent a377ce75

scripts/spdxcheck.py

+57 −10
Original line number Diff line number Diff line
@@ -103,9 +103,21 @@ class id_parser(object): 
        self.spdx_valid = 0

        self.spdx_errors = 0

        self.spdx_dirs = {}

        self.dirdepth = -1

        self.basedir = '.'

        self.curline = 0

        self.deepest = 0



    def set_dirinfo(self, basedir, dirdepth):

        if dirdepth >= 0:

            self.basedir = basedir

            bdir = basedir.lstrip('./').rstrip('/')

            if bdir != '':

                parts = bdir.split('/')

            else:

                parts = []

            self.dirdepth = dirdepth + len(parts)



    # Validate License and Exception IDs

    def validate(self, tok):

        id = tok.value.upper()
@@ -215,12 +227,29 @@ class id_parser(object): 
                sys.stdout.write('%s: %d:0 %s\n' %(fname, self.curline, pe.txt))

            self.spdx_errors += 1



        if fname == '-':

            return



        base = os.path.dirname(fname)

        if self.dirdepth > 0:

            parts = base.split('/')

            i = 0

            base = '.'

            while i < self.dirdepth and i < len(parts) and len(parts[i]):

                base += '/' + parts[i]

                i += 1

        elif self.dirdepth == 0:

            base = self.basedir

        else:

            base = './' + base.rstrip('/')

        base += '/'



        di = self.spdx_dirs.get(base, dirinfo())

        di.update(fail)

        self.spdx_dirs[base] = di



def scan_git_tree(tree):

def scan_git_tree(tree, basedir, dirdepth):

    parser.set_dirinfo(basedir, dirdepth)

    for el in tree.traverse():

        # Exclude stuff which would make pointless noise

        # FIXME: Put this somewhere more sensible
@@ -233,15 +262,19 @@ def scan_git_tree(tree): 
        with open(el.path, 'rb') as fd:

            parser.parse_lines(fd, args.maxlines, el.path)



def scan_git_subtree(tree, path):

def scan_git_subtree(tree, path, dirdepth):

    for p in path.strip('/').split('/'):

        tree = tree[p]

    scan_git_tree(tree)

    scan_git_tree(tree, path.strip('/'), dirdepth)



if __name__ == '__main__':



    ap = ArgumentParser(description='SPDX expression checker')

    ap.add_argument('path', nargs='*', help='Check path or file. If not given full git tree scan. For stdin use "-"')

    ap.add_argument('-d', '--dirs', action='store_true',

                    help='Show [sub]directory statistics.')

    ap.add_argument('-D', '--depth', type=int, default=-1,

                    help='Directory depth for -d statistics. Default: unlimited')

    ap.add_argument('-m', '--maxlines', type=int, default=15,

                    help='Maximum number of lines to scan in a file. Default 15')

    ap.add_argument('-v', '--verbose', action='store_true', help='Verbose statistics output')
@@ -285,13 +318,21 @@ if __name__ == '__main__': 
                    if os.path.isfile(p):

                        parser.parse_lines(open(p, 'rb'), args.maxlines, p)

                    elif os.path.isdir(p):

                        scan_git_subtree(repo.head.reference.commit.tree, p)

                        scan_git_subtree(repo.head.reference.commit.tree, p,

                                         args.depth)

                    else:

                        sys.stderr.write('path %s does not exist\n' %p)

                        sys.exit(1)

            else:

                # Full git tree scan

                scan_git_tree(repo.head.commit.tree)

                scan_git_tree(repo.head.commit.tree, '.', args.depth)



            ndirs = len(parser.spdx_dirs)

            dirsok = 0

            if ndirs:

                for di in parser.spdx_dirs.values():

                    if not di.missing:

                        dirsok += 1



            if args.verbose:

                sys.stderr.write('\n')
@@ -306,17 +347,23 @@ if __name__ == '__main__': 
                    pc = int(100 * parser.spdx_valid / parser.checked)

                    sys.stderr.write('Files with SPDX:   %12d %3d%%\n' %(parser.spdx_valid, pc))

                sys.stderr.write('Files with errors: %12d\n' %parser.spdx_errors)

                ndirs = len(parser.spdx_dirs)

                dirsok = 0

                if ndirs:

                    sys.stderr.write('\n')

                    sys.stderr.write('Directories accounted: %8d\n' %ndirs)

                    for di in parser.spdx_dirs.values():

                        if not di.missing:

                            dirsok += 1

                    pc = int(100 * dirsok / ndirs)

                    sys.stderr.write('Directories complete:  %8d %3d%%\n' %(dirsok, pc))



            if ndirs and ndirs != dirsok and args.dirs:

                if args.verbose:

                    sys.stderr.write('\n')

                sys.stderr.write('Incomplete directories: SPDX in Files\n')

                for f in sorted(parser.spdx_dirs.keys()):

                    di = parser.spdx_dirs[f]

                    if di.missing:

                        valid = di.total - di.missing

                        pc = int(100 * valid / di.total)

                        sys.stderr.write('    %-80s: %5d of %5d  %3d%%\n' %(f, valid, di.total, pc))



            sys.exit(0)



    except Exception as ex: