Add no-strict option to harness

Reviewers: mferencevic

Reviewed By: mferencevic

Differential Revision: https://phabricator.memgraph.io/D716

tests/macro_benchmark/harness/harness.py

@@ -473,6 +473,9 @@ def parse_known_args():
                       help="Logging level")
     argp.add_argument("--additional-run-fields", default={}, type=json.loads,
                       help="Additional fields to add to the 'run', in JSON")
+    argp.add_argument("--no-strict", default=False, action="store_true",
+                      help="Ignores nonexisting groups instead of raising an "
+                           "exception")
     return argp.parse_known_args()
 
 
@@ -510,12 +513,18 @@ def main():
     runner = runners[args.runner](remaining_args)
 
     # Validate groups (if provided)
+    groups = []
     if args.groups:
         for group in args.groups:
             if group not in suite.groups():
-                raise Exception("Group '{}' isn't registered for suite '{}'".
-                                format(group, suite))
-        groups = args.groups
+                msg = "Group '{}' isn't registered for suite '{}'".format(
+                        group, suite)
+                if args.no_strict:
+                    log.warn(msg)
+                else:
+                    raise Exception(msg)
+            else:
+                groups.append(group)
     else:
         # No groups provided, use all suite group
         groups = suite.groups()

tools/apollo/generate

@@ -190,7 +190,7 @@ binary_release_path = os.path.join(BUILD_RELEASE_DIR, binary_release_name)
 binary_release_link_path = os.path.join(BUILD_RELEASE_DIR, "memgraph")
 
 # macro benchmark tests
-MACRO_BENCHMARK_ARGS = "QuerySuite MemgraphRunner --groups aggregation"
+MACRO_BENCHMARK_ARGS = "QuerySuite MemgraphRunner --groups aggregation --no-strict"
 macro_bench_path = os.path.join(BASE_DIR, "tests", "macro_benchmark")
 harness_client_binary = os.path.join(BUILD_RELEASE_DIR, "tests",
         "macro_benchmark", "harness_client")