memgraph/tools/github/clang-tidy/clang-tidy-diff.py

#!/usr/bin/env python3
#
# ===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# ===-----------------------------------------------------------------------===#

r"""
ClangTidy Diff Checker
======================

This script reads input from a unified diff, runs clang-tidy on all changed
files and outputs clang-tidy warnings in changed lines only. This is useful to
detect clang-tidy regressions in the lines touched by a specific patch.
Example usage for git/svn users:

  git diff -U0 HEAD^ | clang-tidy-diff.py -p1
  svn diff --diff-cmd=diff -x-U0 | \
      clang-tidy-diff.py -fix -checks=-*,modernize-use-override

"""

import argparse
import glob
import json
import multiprocessing
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import traceback

try:
    import yaml
except ImportError:
    yaml = None

is_py2 = sys.version[0] == "2"

if is_py2:
    import Queue as queue
else:
    import queue as queue


def run_tidy(task_queue, lock, timeout, exit_status, running_processes):
    watchdog = None
    while True:
        command = task_queue.get()
        with lock:
            should_run = exit_status["status"] == 0
        if not should_run:
            task_queue.task_done()
            continue

        proc = None
        try:
            proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            with lock:
                running_processes.add(proc)

            if timeout is not None:
                watchdog = threading.Timer(timeout, proc.kill)
                watchdog.start()

            stdout, stderr = proc.communicate()
            return_code = proc.returncode

            with lock:
                sys.stdout.write(stdout.decode("utf-8") + "\n")
                sys.stdout.flush()
                if stderr:
                    sys.stderr.write(stderr.decode("utf-8") + "\n")
                    sys.stderr.flush()
                if return_code != 0:
                    exit_status["status"] = 1
                    for p in running_processes:
                        if p != proc:
                            p.terminate()
        except Exception as e:
            with lock:
                sys.stderr.write("Failed: " + str(e) + ": ".join(command) + "\n")
                exit_status["status"] = 1
        finally:
            with lock:
                if not (timeout is None or watchdog is None):
                    if not watchdog.is_alive():
                        sys.stderr.write("Terminated by timeout: " + " ".join(command) + "\n")
                    watchdog.cancel()
                if proc is not None:
                    running_processes.remove(proc)
            task_queue.task_done()


def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout, exit_status, running_processes):
    for _ in range(max_tasks):
        t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout, exit_status, running_processes))
        t.daemon = True
        t.start()


def merge_replacement_files(tmpdir, mergefile):
    """Merge all replacement files in a directory into a single file"""
    # The fixes suggested by clang-tidy >= 4.0.0 are given under
    # the top level key 'Diagnostics' in the output yaml files
    mergekey = "Diagnostics"
    merged = []
    for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")):
        content = yaml.safe_load(open(replacefile, "r"))
        if not content:
            continue  # Skip empty files.
        merged.extend(content.get(mergekey, []))

    if merged:
        # MainSourceFile: The key is required by the definition inside
        # include/clang/Tooling/ReplacementsYaml.h, but the value
        # is actually never used inside clang-apply-replacements,
        # so we set it to '' here.
        output = {"MainSourceFile": "", mergekey: merged}
        with open(mergefile, "w") as out:
            yaml.safe_dump(output, out)
    else:
        # Empty the file:
        open(mergefile, "w").close()


def main():
    parser = argparse.ArgumentParser(
        description="Run clang-tidy against changed files, and " "output diagnostics only for modified " "lines."
    )
    parser.add_argument("-clang-tidy-binary", metavar="PATH", default="clang-tidy", help="path to clang-tidy binary")
    parser.add_argument("-p", metavar="NUM", default=0, help="strip the smallest prefix containing P slashes")
    parser.add_argument(
        "-regex",
        metavar="PATTERN",
        default=None,
        help="custom pattern selecting file paths to check " "(case sensitive, overrides -iregex)",
    )
    parser.add_argument(
        "-iregex",
        metavar="PATTERN",
        default=r".*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)",
        help="custom pattern selecting file paths to check " "(case insensitive, overridden by -regex)",
    )
    parser.add_argument("-j", type=int, default=1, help="number of tidy instances to be run in parallel.")
    parser.add_argument("-timeout", type=int, default=None, help="timeout per each file in seconds.")
    parser.add_argument("-fix", action="store_true", default=False, help="apply suggested fixes")
    parser.add_argument("-checks", help="checks filter, when not specified, use clang-tidy " "default", default="")
    parser.add_argument("-path", dest="build_path", help="Path used to read a compile command database.")
    if yaml:
        parser.add_argument(
            "-export-fixes",
            metavar="FILE",
            dest="export_fixes",
            help="Create a yaml file to store suggested fixes in, "
            "which can be applied with clang-apply-replacements.",
        )
    parser.add_argument(
        "-extra-arg",
        dest="extra_arg",
        action="append",
        default=[],
        help="Additional argument to append to the compiler " "command line.",
    )
    parser.add_argument(
        "-extra-arg-before",
        dest="extra_arg_before",
        action="append",
        default=[],
        help="Additional argument to prepend to the compiler " "command line.",
    )
    parser.add_argument("-quiet", action="store_true", default=False, help="Run clang-tidy in quiet mode")
    clang_tidy_args = []
    argv = sys.argv[1:]
    if "--" in argv:
        clang_tidy_args.extend(argv[argv.index("--") :])
        argv = argv[: argv.index("--")]

    args = parser.parse_args(argv)

    # Extract changed lines for each file.
    filename = None
    lines_by_file = {}
    for line in sys.stdin:
        match = re.search('^\+\+\+\ "?(.*?/){%s}([^ \t\n"]*)' % args.p, line)
        if match:
            filename = match.group(2)
        if filename is None:
            continue

        if args.regex is not None:
            if not re.match("^%s$" % args.regex, filename):
                continue
        else:
            if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE):
                continue

        match = re.search("^@@.*\+(\d+)(,(\d+))?", line)
        if match:
            start_line = int(match.group(1))
            line_count = 1
            if match.group(3):
                line_count = int(match.group(3))
            if line_count == 0:
                continue
            end_line = start_line + line_count - 1
            lines_by_file.setdefault(filename, []).append([start_line, end_line])

    if not any(lines_by_file):
        print("No relevant changes found.")
        sys.exit(0)

    max_task_count = args.j
    if max_task_count == 0:
        max_task_count = multiprocessing.cpu_count()
    max_task_count = min(len(lines_by_file), max_task_count)

    tmpdir = None
    if yaml and args.export_fixes:
        tmpdir = tempfile.mkdtemp()

    # Tasks for clang-tidy.
    task_queue = queue.Queue(max_task_count)
    # A lock for console output.
    lock = threading.Lock()
    running_processes = set()

    exit_status = {"status": 0}

    # Run a pool of clang-tidy workers.
    start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout, exit_status, running_processes)

    # Form the common args list.
    common_clang_tidy_args = []
    if args.fix:
        common_clang_tidy_args.append("-fix")
    if args.checks != "":
        common_clang_tidy_args.append("-checks=" + args.checks)
    if args.quiet:
        common_clang_tidy_args.append("-quiet")
    if args.build_path is not None:
        common_clang_tidy_args.append("-p=%s" % args.build_path)
    for arg in args.extra_arg:
        common_clang_tidy_args.append("-extra-arg=%s" % arg)
    for arg in args.extra_arg_before:
        common_clang_tidy_args.append("-extra-arg-before=%s" % arg)

    for name in lines_by_file:
        line_filter_json = json.dumps([{"name": name, "lines": lines_by_file[name]}], separators=(",", ":"))

        # Run clang-tidy on files containing changes.
        command = [args.clang_tidy_binary]
        command.append("-line-filter=" + line_filter_json)
        if yaml and args.export_fixes:
            # Get a temporary file. We immediately close the handle so clang-tidy can
            # overwrite it.
            (handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir)
            os.close(handle)
            command.append("-export-fixes=" + tmp_name)
        command.extend(common_clang_tidy_args)
        command.append(name)
        command.extend(clang_tidy_args)

        task_queue.put(command)

    # Wait for all threads to be done.
    task_queue.join()

    if exit_status["status"] != 0:
        sys.exit(exit_status["status"])

    if yaml and args.export_fixes:
        print("Writing fixes to " + args.export_fixes + " ...")
        try:
            merge_replacement_files(tmpdir, args.export_fixes)
        except:
            sys.stderr.write("Error exporting fixes.\n")
            traceback.print_exc()

    if tmpdir:
        shutil.rmtree(tmpdir)


if __name__ == "__main__":
    main()