Added 3 iterations to macro benchmark.

Reviewers: mislav.bradac

Reviewed By: mislav.bradac

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D759

tests/macro_benchmark/harness/harness.py

@@ -226,10 +226,6 @@ class _QuerySuite:
                                                     time.time() - start_time))
             return r_val
 
-        measurements = []
-
-        measurement_lists = defaultdict(list)
-
         def add_measurement(dictionary, iteration, key):
             if key in dictionary:
                 measurement = {"target": key,
@@ -243,6 +239,13 @@ class _QuerySuite:
                 except:
                     pass
 
+        measurements = []
+
+        measurement_lists = defaultdict(list)
+
+        # Run the whole test 3 times because memgraph is sometimes
+        # consistently slow and with this hack we get a good median
+        for i in range(3):
             pid = runner.start()
             execute("setup")
 
@@ -282,6 +285,7 @@ class _QuerySuite:
             # TODO value outlier detection and warning across iterations
             execute("teardown")
             runner.stop()
+
         self.append_scenario_summary(group_name, scenario_name,
                                      measurement_lists, num_iterations)
         return measurements