From 5f5d839f0c22a4d611deea0aacce8f171328e454 Mon Sep 17 00:00:00 2001
From: jbajic <jure.bajic@memgraph.com>
Date: Mon, 28 Nov 2022 15:11:58 +0100
Subject: [PATCH] Update analyzer script
---
tests/benchmark/data_structures_contains.cpp | 8 +++---
tests/benchmark/data_structures_find.cpp | 8 +++---
tests/benchmark/data_structures_insert.cpp | 8 +++---
tests/benchmark/data_structures_remove.cpp | 8 +++---
tools/plot/benchmark_datastructures.py | 13 +++++++++
tools/plot/benchmark_datastructures.sh | 29 ++++++++++++++------
6 files changed, 50 insertions(+), 24 deletions(-)
diff --git a/tests/benchmark/data_structures_contains.cpp b/tests/benchmark/data_structures_contains.cpp
index bf1a74a8d..604ab6ef9 100644
--- a/tests/benchmark/data_structures_contains.cpp
+++ b/tests/benchmark/data_structures_contains.cpp
@@ -118,13 +118,13 @@ static void BM_BenchmarkContainsBppTree(::benchmark::State &state) {
state.SetItemsProcessed(found_elems);
}
-BENCHMARK(BM_BenchmarkContainsSkipList)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkContainsSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
-BENCHMARK(BM_BenchmarkContainsStdMap)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkContainsStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
-BENCHMARK(BM_BenchmarkContainsStdSet)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkContainsStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
-BENCHMARK(BM_BenchmarkContainsBppTree)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkContainsBppTree)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
} // namespace memgraph::benchmark
diff --git a/tests/benchmark/data_structures_find.cpp b/tests/benchmark/data_structures_find.cpp
index d3b3bdd60..502be91da 100644
--- a/tests/benchmark/data_structures_find.cpp
+++ b/tests/benchmark/data_structures_find.cpp
@@ -117,13 +117,13 @@ static void BM_BenchmarkFindBppTree(::benchmark::State &state) {
state.SetItemsProcessed(found_elems);
}
-BENCHMARK(BM_BenchmarkFindSkipList)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkFindSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
-BENCHMARK(BM_BenchmarkFindStdMap)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkFindStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
-BENCHMARK(BM_BenchmarkFindStdSet)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkFindStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
-BENCHMARK(BM_BenchmarkFindBppTree)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkFindBppTree)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
} // namespace memgraph::benchmark
diff --git a/tests/benchmark/data_structures_insert.cpp b/tests/benchmark/data_structures_insert.cpp
index 8882b444e..024c23202 100644
--- a/tests/benchmark/data_structures_insert.cpp
+++ b/tests/benchmark/data_structures_insert.cpp
@@ -99,13 +99,13 @@ static void BM_BenchmarkInsertBppTree(::benchmark::State &state) {
}
}
-BENCHMARK(BM_BenchmarkInsertSkipList)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkInsertSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
-BENCHMARK(BM_BenchmarkInsertStdMap)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkInsertStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
-BENCHMARK(BM_BenchmarkInsertStdSet)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkInsertStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
-BENCHMARK(BM_BenchmarkInsertBppTree)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkInsertBppTree)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
} // namespace memgraph::benchmark
diff --git a/tests/benchmark/data_structures_remove.cpp b/tests/benchmark/data_structures_remove.cpp
index 5296a9130..aa02820fc 100644
--- a/tests/benchmark/data_structures_remove.cpp
+++ b/tests/benchmark/data_structures_remove.cpp
@@ -125,13 +125,13 @@ static void BM_BenchmarkRemoveBppTree(::benchmark::State &state) {
state.SetItemsProcessed(removed_elems);
}
-BENCHMARK(BM_BenchmarkRemoveSkipList)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkRemoveSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
-BENCHMARK(BM_BenchmarkRemoveStdMap)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkRemoveStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
-BENCHMARK(BM_BenchmarkRemoveStdSet)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkRemoveStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
-BENCHMARK(BM_BenchmarkRemoveBppTree)->RangeMultiplier(10)->Range(1000, 1000000)->Unit(::benchmark::kMillisecond);
+BENCHMARK(BM_BenchmarkRemoveBppTree)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
} // namespace memgraph::benchmark
diff --git a/tools/plot/benchmark_datastructures.py b/tools/plot/benchmark_datastructures.py
index 29fa61be7..f57516d4d 100644
--- a/tools/plot/benchmark_datastructures.py
+++ b/tools/plot/benchmark_datastructures.py
@@ -9,6 +9,17 @@
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
+####################################
+# Benchmark datastructures analyzer
+####################################
+# This scripts uses the output from dataset benchmark tests to plot charts
+# comparing the results of different datastructures on the same operation.
+#
+# Note: Naming the tests is very important in order for this script to recognize
+# which operation is being performed and on which DS, so it should come in this
+# form: BM_Benchmark<Operation><Datastructure>/<RunArgument>
+# where run_argument will be added automatically by google benchmark framework
+
import argparse
import json
import sys
@@ -142,7 +153,9 @@ def plot_operation(results: GoogleBenchmarkResult, save: bool) -> None:
plt.plot(x_axis, y_axis, marker="", color=colors.pop(0), linewidth="2", label=f"{ds}")
plt.title(f"Benchmark results for operation {results.operation.value}")
plt.xlabel(f"Time [{benchmarks[0].time_unit}]")
+ plt.grid(True)
plt.legend()
+ plt.draw()
else:
print(f"Nothing to do for {ds}...")
if save:
diff --git a/tools/plot/benchmark_datastructures.sh b/tools/plot/benchmark_datastructures.sh
index 450b38466..3001171c0 100755
--- a/tools/plot/benchmark_datastructures.sh
+++ b/tools/plot/benchmark_datastructures.sh
@@ -4,13 +4,26 @@ set -euox pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORKSPACE_DIR=${SCRIPT_DIR}/../../
+CPUS=$(grep -c processor < /proc/cpuinfo)
-BENCHMARK_FILES=$(find ${WORKSPACE_DIR}/tests/benchmark -type f -iname data_structures_*)
-echo $BENCHMARK_FILES
-for bench_file in ${BENCHMARK_FILES}; do
- echo "Running $name"
- base_name=$(basename $bench_file)
- name=${base_name%%.*}
- ${WORKSPACE_DIR}/build/tests/benchmark/${name} --benchmark_format=json --benchmark_out=${name}_output.json
- python3 ${WORKSPACE_DIR}/tools/plot/benchmark_datastructures.py --log-file=${name}_output.json
+# Get all benchmark files
+BENCHMARK_FILES=$(find ${WORKSPACE_DIR}/tests/benchmark -type f -iname "data_structures_*")
+echo $(ls ${WORKSPACE_DIR}/tests/benchmark)
+for BENCH_FILE in ${BENCHMARK_FILES}; do
+ BASE_NAME=$(basename $BENCH_FILE)
+ NAME=${BASE_NAME%%.*}
+ echo "Running $NAME"
+ TEST_FILE=${WORKSPACE_DIR}/build/tests/benchmark/${NAME}
+ if [[ -f "${TEST_FILE}" ]]; then
+ pushd ${WORKSPACE_DIR}/build
+ make -j${CPUS} memgraph__benchmark__${NAME}
+ popd
+ JSON_OUTPUT=${NAME}_output.json
+ # Run benchmakr test
+ ${WORKSPACE_DIR}/build/tests/benchmark/${NAME} --benchmark_format=json --benchmark_out=${JSON_OUTPUT}
+ # Run analyze script for benchmark test
+ python3 ${WORKSPACE_DIR}/tools/plot/benchmark_datastructures.py --log_file=${JSON_OUTPUT}
+ else
+ echo "File ${TEST_FILE} does not exist!"
+ fi
done