diff --git a/tests/benchmark/CMakeLists.txt b/tests/benchmark/CMakeLists.txt index 31f0eebc0..c7cededd8 100644 --- a/tests/benchmark/CMakeLists.txt +++ b/tests/benchmark/CMakeLists.txt @@ -9,11 +9,13 @@ function(add_benchmark test_cpp) get_filename_component(exec_name ${test_cpp} NAME_WE) set(target_name ${test_prefix}${exec_name}) add_executable(${target_name} ${test_cpp} ${ARGN}) + # OUTPUT_NAME sets the real name of a target when it is built and can be # used to help create two targets of the same name even though CMake # requires unique logical target names set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name}) target_link_libraries(${target_name} benchmark gflags) + # register test add_test(${target_name} ${exec_name}) add_dependencies(memgraph__benchmark ${target_name}) @@ -37,9 +39,9 @@ target_link_libraries(${test_prefix}profile mg-query) add_benchmark(query/stripped.cpp) target_link_libraries(${test_prefix}stripped mg-query) -if (MG_ENTERPRISE) -add_benchmark(rpc.cpp) -target_link_libraries(${test_prefix}rpc mg-rpc) +if(MG_ENTERPRISE) + add_benchmark(rpc.cpp) + target_link_libraries(${test_prefix}rpc mg-rpc) endif() add_benchmark(skip_list_random.cpp) @@ -65,3 +67,15 @@ target_link_libraries(${test_prefix}storage_v2_property_store mg-storage-v2) add_benchmark(future.cpp) target_link_libraries(${test_prefix}future mg-io) + +add_benchmark(data_structures_insert.cpp) +target_link_libraries(${test_prefix}data_structures_insert mg-utils mg-storage-v3) + +add_benchmark(data_structures_find.cpp) +target_link_libraries(${test_prefix}data_structures_find mg-utils mg-storage-v3) + +add_benchmark(data_structures_contains.cpp) +target_link_libraries(${test_prefix}data_structures_contains mg-utils mg-storage-v3) + +add_benchmark(data_structures_remove.cpp) +target_link_libraries(${test_prefix}data_structures_remove mg-utils mg-storage-v3) diff --git a/tests/benchmark/data_structures_common.hpp b/tests/benchmark/data_structures_common.hpp new file mode 100644 index 000000000..23fe394ee --- /dev/null +++ b/tests/benchmark/data_structures_common.hpp @@ -0,0 +1,58 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include + +#include "coordinator/hybrid_logical_clock.hpp" +#include "storage/v3/key_store.hpp" +#include "storage/v3/lexicographically_ordered_vertex.hpp" +#include "storage/v3/mvcc.hpp" +#include "storage/v3/transaction.hpp" +#include "utils/skip_list.hpp" + +namespace memgraph::benchmark { + +template +inline void PrepareData(utils::SkipList &skip_list, const int64_t num_elements) { + coordinator::Hlc start_timestamp; + storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION}; + for (auto i{0}; i < num_elements; ++i) { + auto acc = skip_list.access(); + acc.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{true}}}); + } +} + +template +inline void PrepareData(std::map &std_map, const int64_t num_elements) { + coordinator::Hlc start_timestamp; + storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION}; + auto *delta = storage::v3::CreateDeleteObjectDelta(&transaction); + for (auto i{0}; i < num_elements; ++i) { + std_map.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{i}}, + storage::v3::LexicographicallyOrderedVertex{storage::v3::Vertex{ + delta, std::vector{storage::v3::PropertyValue{true}}}}}); + } +} + +template +inline void PrepareData(std::set &std_set, const int64_t num_elements) { + coordinator::Hlc start_timestamp; + storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION}; + for (auto i{0}; i < num_elements; ++i) { + std_set.insert(std::vector{storage::v3::PropertyValue{true}}); + } +} + +} // namespace memgraph::benchmark diff --git a/tests/benchmark/data_structures_contains.cpp b/tests/benchmark/data_structures_contains.cpp new file mode 100644 index 000000000..47af24a1b --- /dev/null +++ b/tests/benchmark/data_structures_contains.cpp @@ -0,0 +1,105 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "data_structures_common.hpp" +#include "storage/v3/key_store.hpp" +#include "storage/v3/lexicographically_ordered_vertex.hpp" +#include "storage/v3/mvcc.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/transaction.hpp" +#include "storage/v3/vertex.hpp" +#include "utils/skip_list.hpp" + +namespace memgraph::benchmark { + +/////////////////////////////////////////////////////////////////////////////// +// Testing Contains Operation +/////////////////////////////////////////////////////////////////////////////// +static void BM_BenchmarkContainsSkipList(::benchmark::State &state) { + utils::SkipList skip_list; + PrepareData(skip_list, state.range(0)); + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t found_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + auto acc = skip_list.access(); + if (acc.contains(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}})) { + found_elems++; + } + } + } + state.SetItemsProcessed(found_elems); +} + +static void BM_BenchmarkContainsStdMap(::benchmark::State &state) { + std::map std_map; + PrepareData(std_map, state.range(0)); + + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t found_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + if (std_map.contains(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}})) { + found_elems++; + } + } + } + state.SetItemsProcessed(found_elems); +} + +static void BM_BenchmarkContainsStdSet(::benchmark::State &state) { + std::set std_set; + PrepareData(std_set, state.range(0)); + + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t found_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + if (std_set.contains(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}})) { + found_elems++; + } + } + } + state.SetItemsProcessed(found_elems); +} + +BENCHMARK(BM_BenchmarkContainsSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkContainsStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkContainsStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +} // namespace memgraph::benchmark + +BENCHMARK_MAIN(); diff --git a/tests/benchmark/data_structures_find.cpp b/tests/benchmark/data_structures_find.cpp new file mode 100644 index 000000000..d51771be5 --- /dev/null +++ b/tests/benchmark/data_structures_find.cpp @@ -0,0 +1,104 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "data_structures_common.hpp" +#include "storage/v3/key_store.hpp" +#include "storage/v3/lexicographically_ordered_vertex.hpp" +#include "storage/v3/mvcc.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/transaction.hpp" +#include "storage/v3/vertex.hpp" +#include "utils/skip_list.hpp" + +namespace memgraph::benchmark { + +/////////////////////////////////////////////////////////////////////////////// +// Testing Find Operation +/////////////////////////////////////////////////////////////////////////////// +static void BM_BenchmarkFindSkipList(::benchmark::State &state) { + utils::SkipList skip_list; + PrepareData(skip_list, state.range(0)); + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t found_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + auto acc = skip_list.access(); + if (acc.find(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}}) != acc.end()) { + found_elems++; + } + } + } + state.SetItemsProcessed(found_elems); +} + +static void BM_BenchmarkFindStdMap(::benchmark::State &state) { + std::map std_map; + PrepareData(std_map, state.range(0)); + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t found_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + if (std_map.find(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}}) != std_map.end()) { + found_elems++; + } + } + } + state.SetItemsProcessed(found_elems); +} + +static void BM_BenchmarkFindStdSet(::benchmark::State &state) { + std::set std_set; + PrepareData(std_set, state.range(0)); + + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t found_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + if (std_set.find(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}}) != std_set.end()) { + found_elems++; + } + } + } + state.SetItemsProcessed(found_elems); +} + +BENCHMARK(BM_BenchmarkFindSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkFindStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkFindStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +} // namespace memgraph::benchmark + +BENCHMARK_MAIN(); diff --git a/tests/benchmark/data_structures_insert.cpp b/tests/benchmark/data_structures_insert.cpp new file mode 100644 index 000000000..d83cbfe23 --- /dev/null +++ b/tests/benchmark/data_structures_insert.cpp @@ -0,0 +1,85 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "storage/v3/key_store.hpp" +#include "storage/v3/lexicographically_ordered_vertex.hpp" +#include "storage/v3/mvcc.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/transaction.hpp" +#include "storage/v3/vertex.hpp" +#include "utils/skip_list.hpp" + +namespace memgraph::benchmark { + +/////////////////////////////////////////////////////////////////////////////// +// Testing Insert Operation +/////////////////////////////////////////////////////////////////////////////// +static void BM_BenchmarkInsertSkipList(::benchmark::State &state) { + utils::SkipList skip_list; + coordinator::Hlc start_timestamp; + storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION}; + + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + auto acc = skip_list.access(); + acc.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{true}}}); + } + } +} + +static void BM_BenchmarkInsertStdMap(::benchmark::State &state) { + std::map std_map; + coordinator::Hlc start_timestamp; + storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION}; + auto *delta = storage::v3::CreateDeleteObjectDelta(&transaction); + + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + std_map.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{i}}, + storage::v3::LexicographicallyOrderedVertex{storage::v3::Vertex{ + delta, std::vector{storage::v3::PropertyValue{true}}}}}); + } + } +} + +static void BM_BenchmarkInsertStdSet(::benchmark::State &state) { + std::set std_set; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + std_set.insert( + storage::v3::PrimaryKey{std::vector{storage::v3::PropertyValue{true}}}); + } + } +} + +BENCHMARK(BM_BenchmarkInsertSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkInsertStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkInsertStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +} // namespace memgraph::benchmark + +BENCHMARK_MAIN(); diff --git a/tests/benchmark/data_structures_remove.cpp b/tests/benchmark/data_structures_remove.cpp new file mode 100644 index 000000000..641ad9453 --- /dev/null +++ b/tests/benchmark/data_structures_remove.cpp @@ -0,0 +1,106 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "data_structures_common.hpp" +#include "storage/v3/key_store.hpp" +#include "storage/v3/lexicographically_ordered_vertex.hpp" +#include "storage/v3/mvcc.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/transaction.hpp" +#include "storage/v3/vertex.hpp" +#include "utils/skip_list.hpp" + +namespace memgraph::benchmark { + +/////////////////////////////////////////////////////////////////////////////// +// Testing Remove Operation +/////////////////////////////////////////////////////////////////////////////// +static void BM_BenchmarkRemoveSkipList(::benchmark::State &state) { + utils::SkipList skip_list; + PrepareData(skip_list, state.range(0)); + + // So we can also have elements that don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t removed_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + auto acc = skip_list.access(); + if (acc.remove(storage::v3::PrimaryKey{storage::v3::PropertyValue(value)})) { + removed_elems++; + } + } + } + state.SetItemsProcessed(removed_elems); +} + +static void BM_BenchmarkRemoveStdMap(::benchmark::State &state) { + std::map std_map; + PrepareData(std_map, state.range(0)); + + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t removed_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + if (std_map.erase(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}}) > 0) { + removed_elems++; + } + } + } + state.SetItemsProcessed(removed_elems); +} + +static void BM_BenchmarkRemoveStdSet(::benchmark::State &state) { + std::set std_set; + PrepareData(std_set, state.range(0)); + + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t removed_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + if (std_set.erase(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}}) > 0) { + removed_elems++; + } + } + } + state.SetItemsProcessed(removed_elems); +} + +BENCHMARK(BM_BenchmarkRemoveSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkRemoveStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkRemoveStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +} // namespace memgraph::benchmark + +BENCHMARK_MAIN(); diff --git a/tests/benchmark/skip_list_common.hpp b/tests/benchmark/skip_list_common.hpp index 988a2dfb7..4b27281c8 100644 --- a/tests/benchmark/skip_list_common.hpp +++ b/tests/benchmark/skip_list_common.hpp @@ -1,4 +1,4 @@ -// Copyright 2021 Memgraph Ltd. +// Copyright 2022 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -11,11 +11,14 @@ #pragma once +#include #include #include +#include #include #include #include +#include #include #include @@ -26,7 +29,7 @@ DEFINE_int32(duration, 10, "Duration of test (in seconds)"); struct Stats { uint64_t total{0}; - uint64_t succ[4] = {0, 0, 0, 0}; + std::array succ = {0, 0, 0, 0}; }; const int OP_INSERT = 0; @@ -81,3 +84,27 @@ inline void RunConcurrentTest(std::function *, Stats *)> std::cout << "Total successful: " << tot << " (" << tot / FLAGS_duration << " calls/s)" << std::endl; std::cout << "Total ops: " << tops << " (" << tops / FLAGS_duration << " calls/s)" << std::endl; } + +inline void RunTest(std::function &, Stats &)> test_func) { + Stats stats; + std::atomic run{true}; + + { + std::jthread bg_thread(test_func, std::cref(run), std::ref(stats)); + std::this_thread::sleep_for(std::chrono::seconds(FLAGS_duration)); + run.store(false, std::memory_order_relaxed); + } + + std::cout << " Operations: " << stats.total << std::endl; + std::cout << " Successful insert: " << stats.succ[0] << std::endl; + std::cout << " Successful contains: " << stats.succ[1] << std::endl; + std::cout << " Successful remove: " << stats.succ[2] << std::endl; + std::cout << " Successful find: " << stats.succ[3] << std::endl; + std::cout << std::endl; + + const auto tot = std::accumulate(stats.succ.begin(), +stats.succ.begin() + 3, 0); + const auto tops = stats.total; + + std::cout << "Total successful: " << tot << " (" << tot / FLAGS_duration << " calls/s)" << std::endl; + std::cout << "Total ops: " << tops << " (" << tops / FLAGS_duration << " calls/s)" << std::endl; +} diff --git a/tools/plot/benchmark_datastructures.py b/tools/plot/benchmark_datastructures.py new file mode 100644 index 000000000..f57516d4d --- /dev/null +++ b/tools/plot/benchmark_datastructures.py @@ -0,0 +1,185 @@ +# Copyright 2022 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +#################################### +# Benchmark datastructures analyzer +#################################### +# This scripts uses the output from dataset benchmark tests to plot charts +# comparing the results of different datastructures on the same operation. +# +# Note: Naming the tests is very important in order for this script to recognize +# which operation is being performed and on which DS, so it should come in this +# form: BM_Benchmark/ +# where run_argument will be added automatically by google benchmark framework + +import argparse +import json +import sys +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Any, Dict, List, Optional + +import matplotlib.pyplot as plt + + +class Operation(Enum): + CONTAINS = "contains" + FIND = "find" + INSERT = "insert" + RANDOM = "random" + REMOVE = "remove" + + @classmethod + def to_list(cls) -> List[str]: + return list(map(lambda c: c.value, cls)) + + @staticmethod + def get(s: str) -> Optional["Operation"]: + try: + return Operation[s.upper()] + except ValueError: + return None + + def __str__(self): + return str(self.value) + + +@dataclass(frozen=True) +class BenchmarkRow: + name: str + datastructure: str + operation: Operation + real_time: int + cpu_time: int + iterations: int + time_unit: str + run_arg: Optional[Any] + + +class GoogleBenchmarkResult: + def __init__(self): + self._operation = None + self._datastructures: Dict[str, List[BenchmarkRow]] = dict() + + def add_result(self, row: BenchmarkRow) -> None: + if self._operation is None: + self._operation = row.operation + assert self._operation is row.operation + if row.datastructure not in self._datastructures: + self._datastructures[row.datastructure] = [row] + else: + self._datastructures[row.datastructure].append(row) + + @property + def operation(self) -> Optional[Operation]: + return self._operation + + @property + def datastructures(self) -> Dict[str, List[BenchmarkRow]]: + return self._datastructures + + +def get_operation(s: str) -> Operation: + for op in Operation.to_list(): + if op.lower() in s.lower(): + operation_enum = Operation.get(op) + if operation_enum is not None: + return operation_enum + else: + print("Operation not found!") + sys.exit(1) + print("Operation not found!") + sys.exit(1) + + +def get_row_data(line: Dict[str, Any]) -> BenchmarkRow: + """ + Naming is very important, first must come an Operation name, and then a data + structure to test. + """ + full_name = line["name"].split("BM_Benchmark")[1] + name_with_run_arg = full_name.split("/") + operation = get_operation(name_with_run_arg[0]) + datastructure = name_with_run_arg[0].split(operation.value.capitalize())[1] + + run_arg = None + if len(name_with_run_arg) > 1: + run_arg = name_with_run_arg[1] + + return BenchmarkRow( + name_with_run_arg[0], + datastructure, + operation, + line["real_time"], + line["cpu_time"], + line["iterations"], + line["time_unit"], + run_arg, + ) + + +def get_benchmark_res(args) -> Optional[GoogleBenchmarkResult]: + file_path = Path(args.log_file) + if not file_path.exists(): + print("Error file {file_path} not found!") + return None + with file_path.open("r") as file: + data = json.load(file) + res = GoogleBenchmarkResult() + assert "benchmarks" in data, "There must be a benchmark list inside" + for benchmark in data["benchmarks"]: + res.add_result(get_row_data(benchmark)) + return res + + +def plot_operation(results: GoogleBenchmarkResult, save: bool) -> None: + colors = ["red", "green", "blue", "yellow", "purple", "brown"] + assert results.operation is not None + fig = plt.figure() + for ds, benchmarks in results.datastructures.items(): + if benchmarks: + # Print line chart + x_axis = [elem.real_time for elem in benchmarks] + y_axis = [elem.run_arg for elem in benchmarks] + plt.plot(x_axis, y_axis, marker="", color=colors.pop(0), linewidth="2", label=f"{ds}") + plt.title(f"Benchmark results for operation {results.operation.value}") + plt.xlabel(f"Time [{benchmarks[0].time_unit}]") + plt.grid(True) + plt.legend() + plt.draw() + else: + print(f"Nothing to do for {ds}...") + if save: + plt.savefig(f"{results.operation.value}.png") + plt.close(fig) + else: + plt.show() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Process benchmark results.") + parser.add_argument("--log_file", type=str) + parser.add_argument("--save", type=bool, default=True) + return parser.parse_args() + + +def main(): + args = parse_args() + res = get_benchmark_res(args) + if res is None: + print("Failed to get results from log file!") + sys.exit(1) + plot_operation(res, args.save) + + +if __name__ == "__main__": + main() diff --git a/tools/plot/benchmark_datastructures.sh b/tools/plot/benchmark_datastructures.sh new file mode 100755 index 000000000..2ab15be31 --- /dev/null +++ b/tools/plot/benchmark_datastructures.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -euox pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +WORKSPACE_DIR=${SCRIPT_DIR}/../../ +CPUS=$(grep -c processor < /proc/cpuinfo) + +# Get all benchmark files +BENCHMARK_FILES=$(find ${WORKSPACE_DIR}/tests/benchmark -type f -iname "data_structures_*") + +function test_all() { + for BENCH_FILE in ${BENCHMARK_FILES[@]}; do + local BASE_NAME=$(basename $BENCH_FILE) + local NAME=${BASE_NAME%%.*} + echo "Running $NAME" + local TEST_FILE=${WORKSPACE_DIR}/build/tests/benchmark/${NAME} + if [[ -f "${TEST_FILE}" ]]; then + pushd ${WORKSPACE_DIR}/build + make -j${CPUS} memgraph__benchmark__${NAME} + popd + local JSON_OUTPUT=${NAME}_output.json + # Run benchmakr test + ${WORKSPACE_DIR}/build/tests/benchmark/${NAME} --benchmark_format=json --benchmark_out=${JSON_OUTPUT} + # Run analyze script for benchmark test + python3 ${WORKSPACE_DIR}/tools/plot/benchmark_datastructures.py --log_file=${JSON_OUTPUT} + else + echo "File ${TEST_FILE} does not exist!" + fi + done +} + +function test_memory() { + ## We are testing only insert + local DATA_STRUCTURES=(SkipList StdMap StdSet BppTree) + for DATA_STRUCTURE in ${DATA_STRUCTURES[@]}; do + valgrind --tool=massif --massif-out-file=${DATA_STRUCTURE}.massif.out ${WORKSPACE_DIR}/build/tests/benchmark/data_structures_insert --benchmark_filter=BM_BenchmarkInsert${DATA_STRUCTURE}/10000 --benchmark_format=json --benchmark_out=${DATA_STRUCTURE}.json + done +} + +ARG_1=${1:-"all"} +case ${ARG_1} in + all) + test_all + ;; + memory) + test_memory + ;; + *) + echo "Select either `all` or `memory` benchmark!" + ;; +esac