Support --benchmarks_filter in the compare.py 'benchmarks' command (#1486)

Previously compare.py ignored the --benchmarks_filter
argument when loading JSON.  This defeated any workflow when
a single run of the benchmark was run, followed by multiple
"subset reports" run against it with the 'benchmarks'
command.

Concretely this came up with the simple case:

 compare.py benchmarks a.json b.json --benchmarks_filter=BM_Example

This has no practical impact on the 'filters' and
'benchmarkfiltered' comand, which do their thing at a later
stage.

Fixes #1484

Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>

tools/compare.py

@@ -9,25 +9,28 @@ import argparse
 from argparse import ArgumentParser
 import json
 import sys
+import os
 import gbench
 from gbench import util, report
-from gbench.util import *
 
 
 def check_inputs(in1, in2, flags):
     """
     Perform checking on the user provided inputs and diagnose any abnormalities
     """
-    in1_kind, in1_err = classify_input_file(in1)
-    in2_kind, in2_err = classify_input_file(in2)
-    output_file = find_benchmark_flag('--benchmark_out=', flags)
-    output_type = find_benchmark_flag('--benchmark_out_format=', flags)
-    if in1_kind == IT_Executable and in2_kind == IT_Executable and output_file:
+    in1_kind, in1_err = util.classify_input_file(in1)
+    in2_kind, in2_err = util.classify_input_file(in2)
+    output_file = util.find_benchmark_flag('--benchmark_out=', flags)
+    output_type = util.find_benchmark_flag('--benchmark_out_format=', flags)
+    if in1_kind == util.IT_Executable and in2_kind == util.IT_Executable and output_file:
         print(("WARNING: '--benchmark_out=%s' will be passed to both "
                "benchmarks causing it to be overwritten") % output_file)
-    if in1_kind == IT_JSON and in2_kind == IT_JSON and len(flags) > 0:
-        print("WARNING: passing optional flags has no effect since both "
-              "inputs are JSON")
+    if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON:
+        # When both sides are JSON the only supported flag is
+        # --benchmark_filter=
+        for flag in util.remove_benchmark_flags('--benchmark_filter=', flags):
+            print("WARNING: passing %s has no effect since both "
+                  "inputs are JSON" % flag)
     if output_type is not None and output_type != 'json':
         print(("ERROR: passing '--benchmark_out_format=%s' to 'compare.py`"
                " is not supported.") % output_type)

tools/gbench/util.py

@@ -2,10 +2,11 @@
 """
 import json
 import os
-import tempfile
+import re
 import subprocess
 import sys
-import functools
+import tempfile
+
 
 # Input file type enumeration
 IT_Invalid = 0
@@ -111,13 +112,32 @@ def remove_benchmark_flags(prefix, benchmark_flags):
     return [f for f in benchmark_flags if not f.startswith(prefix)]
 
 
-def load_benchmark_results(fname):
+def load_benchmark_results(fname, benchmark_filter):
     """
     Read benchmark output from a file and return the JSON object.
+
+    Apply benchmark_filter, a regular expression, with nearly the same
+    semantics of the --benchmark_filter argument.  May be None.
+    Note: the Python regular expression engine is used instead of the
+    one used by the C++ code, which may produce different results
+    in complex cases.
+
     REQUIRES: 'fname' names a file containing JSON benchmark output.
     """
+    def benchmark_wanted(benchmark):
+        if benchmark_filter is None:
+            return True
+        name = benchmark.get('run_name', None) or benchmark['name']
+        if re.search(benchmark_filter, name):
+            return True
+        return False
+
     with open(fname, 'r') as f:
-        return json.load(f)
+        results = json.load(f)
+        if 'benchmarks' in results:
+            results['benchmarks'] = list(filter(benchmark_wanted,
+                                                results['benchmarks']))
+        return results
 
 
 def sort_benchmark_results(result):
@@ -160,7 +180,7 @@ def run_benchmark(exe_name, benchmark_flags):
     if exitCode != 0:
         print('TEST FAILED...')
         sys.exit(exitCode)
-    json_res = load_benchmark_results(output_name)
+    json_res = load_benchmark_results(output_name, None)
     if is_temp_output:
         os.unlink(output_name)
     return json_res
@@ -175,7 +195,9 @@ def run_or_load_benchmark(filename, benchmark_flags):
     """
     ftype = check_input_file(filename)
     if ftype == IT_JSON:
-        return load_benchmark_results(filename)
+        benchmark_filter = find_benchmark_flag('--benchmark_filter=',
+                                               benchmark_flags)
+        return load_benchmark_results(filename, benchmark_filter)
     if ftype == IT_Executable:
         return run_benchmark(filename, benchmark_flags)
     raise ValueError('Unknown file type %s' % ftype)