Added max memory usage to harness

Reviewers: buda, mislav.bradac

Reviewed By: mislav.bradac

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D790

tests/macro_benchmark/harness/clients.py

@@ -3,7 +3,7 @@ import os
 import time
 import json
 import tempfile
-from common import get_absolute_path, WALL_TIME, CPU_TIME
+from common import get_absolute_path, WALL_TIME, CPU_TIME, MAX_MEMORY
 
 log = logging.getLogger(__name__)
 
@@ -56,7 +56,8 @@ class QueryClient:
         # TODO make the timeout configurable per query or something
         return_code = self.client.run_and_wait(
             client, client_args, timeout=600, stdin=queries_path)
-        cpu_time_end = database.database_bin.get_usage()["cpu"]
+        usage = database.database_bin.get_usage()
+        cpu_time_end = usage["cpu"]
         os.remove(queries_path)
         if return_code != 0:
             with open(self.client.get_stderr()) as f:
@@ -69,6 +70,7 @@ class QueryClient:
         with open(output) as f:
             data = json.loads(f.read())
         data[CPU_TIME] = cpu_time_end - cpu_time_start
+        data[MAX_MEMORY] = usage["max_memory"]
 
         os.remove(output)
         return data

tests/macro_benchmark/harness/common.py

@@ -2,6 +2,7 @@ import os
 
 WALL_TIME = "wall_time"
 CPU_TIME = "cpu_time"
+MAX_MEMORY = "max_memory"
 
 DIR_PATH = os.path.dirname(os.path.realpath(__file__))
 

tests/macro_benchmark/harness/query_suite.py

@@ -7,7 +7,7 @@ from argparse import ArgumentParser
 from collections import defaultdict
 import tempfile
 from statistics import median
-from common import get_absolute_path, WALL_TIME, CPU_TIME
+from common import get_absolute_path, WALL_TIME, CPU_TIME, MAX_MEMORY
 from databases import Memgraph, Neo
 from clients import QueryClient
 
@@ -25,12 +25,12 @@ class _QuerySuite:
     KNOWN_KEYS = {"config", "setup", "itersetup", "run", "iterteardown",
                   "teardown", "common"}
     FORMAT = ["{:>24}", "{:>28}", "{:>16}", "{:>18}", "{:>22}",
-              "{:>16}", "{:>16}"]
+              "{:>16}", "{:>16}", "{:>16}"]
     FULL_FORMAT = "".join(FORMAT) + "\n"
     summary = FULL_FORMAT.format(
                       "group_name", "scenario_name", "parsing_time",
                       "planning_time", "plan_execution_time",
-                      WALL_TIME, CPU_TIME)
+                      WALL_TIME, CPU_TIME, MAX_MEMORY)
 
     def __init__(self, args):
         pass
@@ -93,6 +93,7 @@ class _QuerySuite:
                                      scenario_config.get("num_client_workers", 1))
                 add_measurement(run_result, iteration, WALL_TIME)
                 add_measurement(run_result, iteration, CPU_TIME)
+                add_measurement(run_result, iteration, MAX_MEMORY)
                 for measurement in ["parsing_time",
                                     "plan_execution_time",
                                     "planning_time"] :
@@ -114,12 +115,17 @@ class _QuerySuite:
         self.summary += self.FORMAT[0].format(group_name)
         self.summary += self.FORMAT[1].format(scenario_name)
         for i, key in enumerate(("parsing_time", "planning_time",
-                    "plan_execution_time", WALL_TIME, CPU_TIME)):
+                    "plan_execution_time", WALL_TIME, CPU_TIME, MAX_MEMORY)):
             if key not in measurement_lists:
                 time = "-"
             else:
+                fmt = "{:.10f}"
                 # Median is used instead of avg to avoid effect of outliers.
-                time = "{:.10f}".format(median(measurement_lists[key]))
+                value = median(measurement_lists[key])
+                if key == MAX_MEMORY:
+                    fmt = "{}"
+                    value = int(value)
+                time = fmt.format(value)
             self.summary += self.FORMAT[i + 2].format(time)
         self.summary += "\n"
 

tools/apollo/macro_benchmark_summary

@@ -51,20 +51,28 @@ def compare_values(data_cur, data_prev):
                 item = " ".join(item_cur.split("_")).capitalize()
             else:
                 item_prev = find_item(data_prev, headers[j], row_cur)
+                if j != len(row_cur) - 1:
+                    fmt = "{:.3f}ms"
+                    scale = 1000.0
+                else:
+                    fmt = "{:.2f}MiB"
+                    scale = 1.0 / 1024.0
                 if item_prev != None:
                     if item_prev != 0.0:
                         diff = (item_cur - item_prev) / item_prev
                     else:
                         diff = 0.0
-                    if diff < -0.05:
+                    if diff < -0.05 and item_cur > 0.0005:
                         sign = " {icon arrow-down color=green}"
-                    elif diff > 0.05:
+                    elif diff > 0.05 and item_cur > 0.0005:
                         sign = " {icon arrow-up color=red}"
                     else:
                         sign = ""
-                    item = "{:.3f}ms //({:+.2%})//{}".format(item_cur * 1000.0, diff, sign)
+                    fmt += " //({:+.2%})//{}"
+                    item = fmt.format(item_cur * scale, diff, sign)
                 else:
-                    item = "{:.3f}ms //(new)// {{icon plus color=blue}}".format(item_cur * 1000.0)
+                    fmt += "//(new)// {{icon plus color=blue}}"
+                    item = fmt.format(item_cur * scale)
             ret[i].append(item)
     return ret