diff --git a/tests/macro_benchmark/harness/clients.py b/tests/macro_benchmark/harness/clients.py index f61f2a7d9..38ca70978 100644 --- a/tests/macro_benchmark/harness/clients.py +++ b/tests/macro_benchmark/harness/clients.py @@ -3,7 +3,7 @@ import os import time import json import tempfile -from common import get_absolute_path, WALL_TIME, CPU_TIME +from common import get_absolute_path, WALL_TIME, CPU_TIME, MAX_MEMORY log = logging.getLogger(__name__) @@ -56,7 +56,8 @@ class QueryClient: # TODO make the timeout configurable per query or something return_code = self.client.run_and_wait( client, client_args, timeout=600, stdin=queries_path) - cpu_time_end = database.database_bin.get_usage()["cpu"] + usage = database.database_bin.get_usage() + cpu_time_end = usage["cpu"] os.remove(queries_path) if return_code != 0: with open(self.client.get_stderr()) as f: @@ -69,6 +70,7 @@ class QueryClient: with open(output) as f: data = json.loads(f.read()) data[CPU_TIME] = cpu_time_end - cpu_time_start + data[MAX_MEMORY] = usage["max_memory"] os.remove(output) return data diff --git a/tests/macro_benchmark/harness/common.py b/tests/macro_benchmark/harness/common.py index e5cab0d41..12ce208a1 100644 --- a/tests/macro_benchmark/harness/common.py +++ b/tests/macro_benchmark/harness/common.py @@ -2,6 +2,7 @@ import os WALL_TIME = "wall_time" CPU_TIME = "cpu_time" +MAX_MEMORY = "max_memory" DIR_PATH = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/macro_benchmark/harness/query_suite.py b/tests/macro_benchmark/harness/query_suite.py index b87b4883a..ce42891a2 100644 --- a/tests/macro_benchmark/harness/query_suite.py +++ b/tests/macro_benchmark/harness/query_suite.py @@ -7,7 +7,7 @@ from argparse import ArgumentParser from collections import defaultdict import tempfile from statistics import median -from common import get_absolute_path, WALL_TIME, CPU_TIME +from common import get_absolute_path, WALL_TIME, CPU_TIME, MAX_MEMORY from databases import Memgraph, Neo from clients import QueryClient @@ -25,12 +25,12 @@ class _QuerySuite: KNOWN_KEYS = {"config", "setup", "itersetup", "run", "iterteardown", "teardown", "common"} FORMAT = ["{:>24}", "{:>28}", "{:>16}", "{:>18}", "{:>22}", - "{:>16}", "{:>16}"] + "{:>16}", "{:>16}", "{:>16}"] FULL_FORMAT = "".join(FORMAT) + "\n" summary = FULL_FORMAT.format( "group_name", "scenario_name", "parsing_time", "planning_time", "plan_execution_time", - WALL_TIME, CPU_TIME) + WALL_TIME, CPU_TIME, MAX_MEMORY) def __init__(self, args): pass @@ -93,6 +93,7 @@ class _QuerySuite: scenario_config.get("num_client_workers", 1)) add_measurement(run_result, iteration, WALL_TIME) add_measurement(run_result, iteration, CPU_TIME) + add_measurement(run_result, iteration, MAX_MEMORY) for measurement in ["parsing_time", "plan_execution_time", "planning_time"] : @@ -114,12 +115,17 @@ class _QuerySuite: self.summary += self.FORMAT[0].format(group_name) self.summary += self.FORMAT[1].format(scenario_name) for i, key in enumerate(("parsing_time", "planning_time", - "plan_execution_time", WALL_TIME, CPU_TIME)): + "plan_execution_time", WALL_TIME, CPU_TIME, MAX_MEMORY)): if key not in measurement_lists: time = "-" else: + fmt = "{:.10f}" # Median is used instead of avg to avoid effect of outliers. - time = "{:.10f}".format(median(measurement_lists[key])) + value = median(measurement_lists[key]) + if key == MAX_MEMORY: + fmt = "{}" + value = int(value) + time = fmt.format(value) self.summary += self.FORMAT[i + 2].format(time) self.summary += "\n" diff --git a/tools/apollo/macro_benchmark_summary b/tools/apollo/macro_benchmark_summary index 2400c81bf..414266fb8 100755 --- a/tools/apollo/macro_benchmark_summary +++ b/tools/apollo/macro_benchmark_summary @@ -51,20 +51,28 @@ def compare_values(data_cur, data_prev): item = " ".join(item_cur.split("_")).capitalize() else: item_prev = find_item(data_prev, headers[j], row_cur) + if j != len(row_cur) - 1: + fmt = "{:.3f}ms" + scale = 1000.0 + else: + fmt = "{:.2f}MiB" + scale = 1.0 / 1024.0 if item_prev != None: if item_prev != 0.0: diff = (item_cur - item_prev) / item_prev else: diff = 0.0 - if diff < -0.05: + if diff < -0.05 and item_cur > 0.0005: sign = " {icon arrow-down color=green}" - elif diff > 0.05: + elif diff > 0.05 and item_cur > 0.0005: sign = " {icon arrow-up color=red}" else: sign = "" - item = "{:.3f}ms //({:+.2%})//{}".format(item_cur * 1000.0, diff, sign) + fmt += " //({:+.2%})//{}" + item = fmt.format(item_cur * scale, diff, sign) else: - item = "{:.3f}ms //(new)// {{icon plus color=blue}}".format(item_cur * 1000.0) + fmt += "//(new)// {{icon plus color=blue}}" + item = fmt.format(item_cur * scale) ret[i].append(item) return ret