tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #668 from memgraph/T1173-MG-benchmark-datastructures

Browse Source 
- Add benchmarks comparing insert, remove, find and contains for std::map, std::set and utils::SkipList- 
- Create a python plot script
- Create a bash helper script for executing and plotting benchmarks
This commit is contained in:
Jure Bajic 2022-12-12 04:11:55 -08:00 committed by GitHub
commit 03d994318e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 741 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
tests/benchmark/CMakeLists.txt
View File

@ -9,11 +9,13 @@ function(add_benchmark test_cpp)
get_filename_component(exec_name ${test_cpp} NAME_WE)
set(target_name ${test_prefix}${exec_name})
add_executable(${target_name} ${test_cpp} ${ARGN})
# OUTPUT_NAME sets the real name of a target when it is built and can be
# used to help create two targets of the same name even though CMake
# requires unique logical target names
set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name})
target_link_libraries(${target_name} benchmark gflags)
# register test
add_test(${target_name} ${exec_name})
add_dependencies(memgraph__benchmark ${target_name})
@ -37,9 +39,9 @@ target_link_libraries(${test_prefix}profile mg-query)
add_benchmark(query/stripped.cpp)
target_link_libraries(${test_prefix}stripped mg-query)
if (MG_ENTERPRISE)
add_benchmark(rpc.cpp)
target_link_libraries(${test_prefix}rpc mg-rpc)
if(MG_ENTERPRISE)
add_benchmark(rpc.cpp)
target_link_libraries(${test_prefix}rpc mg-rpc)
endif()
add_benchmark(skip_list_random.cpp)
@ -65,3 +67,15 @@ target_link_libraries(${test_prefix}storage_v2_property_store mg-storage-v2)
add_benchmark(future.cpp)
target_link_libraries(${test_prefix}future mg-io)
add_benchmark(data_structures_insert.cpp)
target_link_libraries(${test_prefix}data_structures_insert mg-utils mg-storage-v3)
add_benchmark(data_structures_find.cpp)
target_link_libraries(${test_prefix}data_structures_find mg-utils mg-storage-v3)
add_benchmark(data_structures_contains.cpp)
target_link_libraries(${test_prefix}data_structures_contains mg-utils mg-storage-v3)
add_benchmark(data_structures_remove.cpp)
target_link_libraries(${test_prefix}data_structures_remove mg-utils mg-storage-v3)

58
tests/benchmark/data_structures_common.hpp Normal file
View File

@ -0,0 +1,58 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <map>
#include <set>
#include <vector>
#include "coordinator/hybrid_logical_clock.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/lexicographically_ordered_vertex.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/transaction.hpp"
#include "utils/skip_list.hpp"
namespace memgraph::benchmark {
template <typename T>
inline void PrepareData(utils::SkipList<T> &skip_list, const int64_t num_elements) {
coordinator::Hlc start_timestamp;
storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION};
for (auto i{0}; i < num_elements; ++i) {
auto acc = skip_list.access();
acc.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{true}}});
}
}
template <typename TKey, typename TValue>
inline void PrepareData(std::map<TKey, TValue> &std_map, const int64_t num_elements) {
coordinator::Hlc start_timestamp;
storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION};
auto *delta = storage::v3::CreateDeleteObjectDelta(&transaction);
for (auto i{0}; i < num_elements; ++i) {
std_map.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{i}},
storage::v3::LexicographicallyOrderedVertex{storage::v3::Vertex{
delta, std::vector<storage::v3::PropertyValue>{storage::v3::PropertyValue{true}}}}});
}
}
template <typename T>
inline void PrepareData(std::set<T> &std_set, const int64_t num_elements) {
coordinator::Hlc start_timestamp;
storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION};
for (auto i{0}; i < num_elements; ++i) {
std_set.insert(std::vector<storage::v3::PropertyValue>{storage::v3::PropertyValue{true}});
}
}
} // namespace memgraph::benchmark

105
tests/benchmark/data_structures_contains.cpp Normal file
View File

@ -0,0 +1,105 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <atomic>
#include <concepts>
#include <cstddef>
#include <cstdint>
#include <exception>
#include <map>
#include <set>
#include <stdexcept>
#include <type_traits>
#include <vector>
#include <benchmark/benchmark.h>
#include <gflags/gflags.h>
#include "data_structures_common.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/lexicographically_ordered_vertex.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "utils/skip_list.hpp"
namespace memgraph::benchmark {
///////////////////////////////////////////////////////////////////////////////
// Testing Contains Operation
///////////////////////////////////////////////////////////////////////////////
static void BM_BenchmarkContainsSkipList(::benchmark::State &state) {
utils::SkipList<storage::v3::PrimaryKey> skip_list;
PrepareData(skip_list, state.range(0));
// So we can also have elements that does don't exist
std::mt19937 i_generator(std::random_device{}());
std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
int64_t found_elems{0};
for (auto _ : state) {
for (auto i{0}; i < state.range(0); ++i) {
int64_t value = i_distribution(i_generator);
auto acc = skip_list.access();
if (acc.contains(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}})) {
found_elems++;
}
}
}
state.SetItemsProcessed(found_elems);
}
static void BM_BenchmarkContainsStdMap(::benchmark::State &state) {
std::map<storage::v3::PrimaryKey, storage::v3::LexicographicallyOrderedVertex> std_map;
PrepareData(std_map, state.range(0));
// So we can also have elements that does don't exist
std::mt19937 i_generator(std::random_device{}());
std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
int64_t found_elems{0};
for (auto _ : state) {
for (auto i{0}; i < state.range(0); ++i) {
int64_t value = i_distribution(i_generator);
if (std_map.contains(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}})) {
found_elems++;
}
}
}
state.SetItemsProcessed(found_elems);
}
static void BM_BenchmarkContainsStdSet(::benchmark::State &state) {
std::set<storage::v3::PrimaryKey> std_set;
PrepareData(std_set, state.range(0));
// So we can also have elements that does don't exist
std::mt19937 i_generator(std::random_device{}());
std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
int64_t found_elems{0};
for (auto _ : state) {
for (auto i{0}; i < state.range(0); ++i) {
int64_t value = i_distribution(i_generator);
if (std_set.contains(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}})) {
found_elems++;
}
}
}
state.SetItemsProcessed(found_elems);
}
BENCHMARK(BM_BenchmarkContainsSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
BENCHMARK(BM_BenchmarkContainsStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
BENCHMARK(BM_BenchmarkContainsStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
} // namespace memgraph::benchmark
BENCHMARK_MAIN();

104
tests/benchmark/data_structures_find.cpp Normal file
View File

@ -0,0 +1,104 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <atomic>
#include <concepts>
#include <cstddef>
#include <cstdint>
#include <exception>
#include <map>
#include <set>
#include <stdexcept>
#include <type_traits>
#include <vector>
#include <benchmark/benchmark.h>
#include <gflags/gflags.h>
#include "data_structures_common.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/lexicographically_ordered_vertex.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "utils/skip_list.hpp"
namespace memgraph::benchmark {
///////////////////////////////////////////////////////////////////////////////
// Testing Find Operation
///////////////////////////////////////////////////////////////////////////////
static void BM_BenchmarkFindSkipList(::benchmark::State &state) {
utils::SkipList<storage::v3::PrimaryKey> skip_list;
PrepareData(skip_list, state.range(0));
// So we can also have elements that does don't exist
std::mt19937 i_generator(std::random_device{}());
std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
int64_t found_elems{0};
for (auto _ : state) {
for (auto i{0}; i < state.range(0); ++i) {
int64_t value = i_distribution(i_generator);
auto acc = skip_list.access();
if (acc.find(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}}) != acc.end()) {
found_elems++;
}
}
}
state.SetItemsProcessed(found_elems);
}
static void BM_BenchmarkFindStdMap(::benchmark::State &state) {
std::map<storage::v3::PrimaryKey, storage::v3::LexicographicallyOrderedVertex> std_map;
PrepareData(std_map, state.range(0));
// So we can also have elements that does don't exist
std::mt19937 i_generator(std::random_device{}());
std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
int64_t found_elems{0};
for (auto _ : state) {
for (auto i{0}; i < state.range(0); ++i) {
int64_t value = i_distribution(i_generator);
if (std_map.find(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}}) != std_map.end()) {
found_elems++;
}
}
}
state.SetItemsProcessed(found_elems);
}
static void BM_BenchmarkFindStdSet(::benchmark::State &state) {
std::set<storage::v3::PrimaryKey> std_set;
PrepareData(std_set, state.range(0));
// So we can also have elements that does don't exist
std::mt19937 i_generator(std::random_device{}());
std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
int64_t found_elems{0};
for (auto _ : state) {
for (auto i{0}; i < state.range(0); ++i) {
int64_t value = i_distribution(i_generator);
if (std_set.find(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}}) != std_set.end()) {
found_elems++;
}
}
}
state.SetItemsProcessed(found_elems);
}
BENCHMARK(BM_BenchmarkFindSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
BENCHMARK(BM_BenchmarkFindStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
BENCHMARK(BM_BenchmarkFindStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
} // namespace memgraph::benchmark
BENCHMARK_MAIN();

85
tests/benchmark/data_structures_insert.cpp Normal file
View File

@ -0,0 +1,85 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <atomic>
#include <concepts>
#include <cstddef>
#include <cstdint>
#include <exception>
#include <map>
#include <set>
#include <stdexcept>
#include <type_traits>
#include <vector>
#include <benchmark/benchmark.h>
#include <gflags/gflags.h>
#include "storage/v3/key_store.hpp"
#include "storage/v3/lexicographically_ordered_vertex.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "utils/skip_list.hpp"
namespace memgraph::benchmark {
///////////////////////////////////////////////////////////////////////////////
// Testing Insert Operation
///////////////////////////////////////////////////////////////////////////////
static void BM_BenchmarkInsertSkipList(::benchmark::State &state) {
utils::SkipList<storage::v3::PrimaryKey> skip_list;
coordinator::Hlc start_timestamp;
storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION};
for (auto _ : state) {
for (auto i{0}; i < state.range(0); ++i) {
auto acc = skip_list.access();
acc.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{true}}});
}
}
}
static void BM_BenchmarkInsertStdMap(::benchmark::State &state) {
std::map<storage::v3::PrimaryKey, storage::v3::LexicographicallyOrderedVertex> std_map;
coordinator::Hlc start_timestamp;
storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION};
auto *delta = storage::v3::CreateDeleteObjectDelta(&transaction);
for (auto _ : state) {
for (auto i{0}; i < state.range(0); ++i) {
std_map.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{i}},
storage::v3::LexicographicallyOrderedVertex{storage::v3::Vertex{
delta, std::vector<storage::v3::PropertyValue>{storage::v3::PropertyValue{true}}}}});
}
}
}
static void BM_BenchmarkInsertStdSet(::benchmark::State &state) {
std::set<storage::v3::PrimaryKey> std_set;
for (auto _ : state) {
for (auto i{0}; i < state.range(0); ++i) {
std_set.insert(
storage::v3::PrimaryKey{std::vector<storage::v3::PropertyValue>{storage::v3::PropertyValue{true}}});
}
}
}
BENCHMARK(BM_BenchmarkInsertSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
BENCHMARK(BM_BenchmarkInsertStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
BENCHMARK(BM_BenchmarkInsertStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
} // namespace memgraph::benchmark
BENCHMARK_MAIN();

106
tests/benchmark/data_structures_remove.cpp Normal file
View File

@ -0,0 +1,106 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <atomic>
#include <concepts>
#include <cstddef>
#include <cstdint>
#include <exception>
#include <map>
#include <set>
#include <stdexcept>
#include <type_traits>
#include <vector>
#include <benchmark/benchmark.h>
#include <gflags/gflags.h>
#include "data_structures_common.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/lexicographically_ordered_vertex.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "utils/skip_list.hpp"
namespace memgraph::benchmark {
///////////////////////////////////////////////////////////////////////////////
// Testing Remove Operation
///////////////////////////////////////////////////////////////////////////////
static void BM_BenchmarkRemoveSkipList(::benchmark::State &state) {
utils::SkipList<storage::v3::PrimaryKey> skip_list;
PrepareData(skip_list, state.range(0));
// So we can also have elements that don't exist
std::mt19937 i_generator(std::random_device{}());
std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
int64_t removed_elems{0};
for (auto _ : state) {
for (auto i{0}; i < state.range(0); ++i) {
int64_t value = i_distribution(i_generator);
auto acc = skip_list.access();
if (acc.remove(storage::v3::PrimaryKey{storage::v3::PropertyValue(value)})) {
removed_elems++;
}
}
}
state.SetItemsProcessed(removed_elems);
}
static void BM_BenchmarkRemoveStdMap(::benchmark::State &state) {
std::map<storage::v3::PrimaryKey, storage::v3::LexicographicallyOrderedVertex> std_map;
PrepareData(std_map, state.range(0));
// So we can also have elements that does don't exist
std::mt19937 i_generator(std::random_device{}());
std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
int64_t removed_elems{0};
for (auto _ : state) {
for (auto i{0}; i < state.range(0); ++i) {
int64_t value = i_distribution(i_generator);
if (std_map.erase(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}}) > 0) {
removed_elems++;
}
}
}
state.SetItemsProcessed(removed_elems);
}
static void BM_BenchmarkRemoveStdSet(::benchmark::State &state) {
std::set<storage::v3::PrimaryKey> std_set;
PrepareData(std_set, state.range(0));
// So we can also have elements that does don't exist
std::mt19937 i_generator(std::random_device{}());
std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
int64_t removed_elems{0};
for (auto _ : state) {
for (auto i{0}; i < state.range(0); ++i) {
int64_t value = i_distribution(i_generator);
if (std_set.erase(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}}) > 0) {
removed_elems++;
}
}
}
state.SetItemsProcessed(removed_elems);
}
BENCHMARK(BM_BenchmarkRemoveSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
BENCHMARK(BM_BenchmarkRemoveStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
BENCHMARK(BM_BenchmarkRemoveStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
} // namespace memgraph::benchmark
BENCHMARK_MAIN();

31
tests/benchmark/skip_list_common.hpp
View File

@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -11,11 +11,14 @@
#pragma once
#include <array>
#include <atomic>
#include <chrono>
#include <cstdint>
#include <functional>
#include <iostream>
#include <memory>
#include <numeric>
#include <thread>
#include <vector>
@ -26,7 +29,7 @@ DEFINE_int32(duration, 10, "Duration of test (in seconds)");
struct Stats {
uint64_t total{0};
uint64_t succ[4] = {0, 0, 0, 0};
std::array<uint64_t, 4> succ = {0, 0, 0, 0};
};
const int OP_INSERT = 0;
@ -81,3 +84,27 @@ inline void RunConcurrentTest(std::function<void(std::atomic<bool> *, Stats *)>
std::cout << "Total successful: " << tot << " (" << tot / FLAGS_duration << " calls/s)" << std::endl;
std::cout << "Total ops: " << tops << " (" << tops / FLAGS_duration << " calls/s)" << std::endl;
}
inline void RunTest(std::function<void(const std::atomic<bool> &, Stats &)> test_func) {
Stats stats;
std::atomic<bool> run{true};
{
std::jthread bg_thread(test_func, std::cref(run), std::ref(stats));
std::this_thread::sleep_for(std::chrono::seconds(FLAGS_duration));
run.store(false, std::memory_order_relaxed);
}
std::cout << " Operations: " << stats.total << std::endl;
std::cout << " Successful insert: " << stats.succ[0] << std::endl;
std::cout << " Successful contains: " << stats.succ[1] << std::endl;
std::cout << " Successful remove: " << stats.succ[2] << std::endl;
std::cout << " Successful find: " << stats.succ[3] << std::endl;
std::cout << std::endl;
const auto tot = std::accumulate(stats.succ.begin(), +stats.succ.begin() + 3, 0);
const auto tops = stats.total;
std::cout << "Total successful: " << tot << " (" << tot / FLAGS_duration << " calls/s)" << std::endl;
std::cout << "Total ops: " << tops << " (" << tops / FLAGS_duration << " calls/s)" << std::endl;
}

185
tools/plot/benchmark_datastructures.py Normal file
View File

@ -0,0 +1,185 @@
# Copyright 2022 Memgraph Ltd.
#
# Use of this software is governed by the Business Source License
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
# License, and you may not use this file except in compliance with the Business Source License.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
####################################
# Benchmark datastructures analyzer
####################################
# This scripts uses the output from dataset benchmark tests to plot charts
# comparing the results of different datastructures on the same operation.
#
# Note: Naming the tests is very important in order for this script to recognize
# which operation is being performed and on which DS, so it should come in this
# form: BM_Benchmark<Operation><Datastructure>/<RunArgument>
# where run_argument will be added automatically by google benchmark framework
import argparse
import json
import sys
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from typing import Any, Dict, List, Optional
import matplotlib.pyplot as plt
class Operation(Enum):
CONTAINS = "contains"
FIND = "find"
INSERT = "insert"
RANDOM = "random"
REMOVE = "remove"
@classmethod
def to_list(cls) -> List[str]:
return list(map(lambda c: c.value, cls))
@staticmethod
def get(s: str) -> Optional["Operation"]:
try:
return Operation[s.upper()]
except ValueError:
return None
def __str__(self):
return str(self.value)
@dataclass(frozen=True)
class BenchmarkRow:
name: str
datastructure: str
operation: Operation
real_time: int
cpu_time: int
iterations: int
time_unit: str
run_arg: Optional[Any]
class GoogleBenchmarkResult:
def __init__(self):
self._operation = None
self._datastructures: Dict[str, List[BenchmarkRow]] = dict()
def add_result(self, row: BenchmarkRow) -> None:
if self._operation is None:
self._operation = row.operation
assert self._operation is row.operation
if row.datastructure not in self._datastructures:
self._datastructures[row.datastructure] = [row]
else:
self._datastructures[row.datastructure].append(row)
@property
def operation(self) -> Optional[Operation]:
return self._operation
@property
def datastructures(self) -> Dict[str, List[BenchmarkRow]]:
return self._datastructures
def get_operation(s: str) -> Operation:
for op in Operation.to_list():
if op.lower() in s.lower():
operation_enum = Operation.get(op)
if operation_enum is not None:
return operation_enum
else:
print("Operation not found!")
sys.exit(1)
print("Operation not found!")
sys.exit(1)
def get_row_data(line: Dict[str, Any]) -> BenchmarkRow:
"""
Naming is very important, first must come an Operation name, and then a data
structure to test.
"""
full_name = line["name"].split("BM_Benchmark")[1]
name_with_run_arg = full_name.split("/")
operation = get_operation(name_with_run_arg[0])
datastructure = name_with_run_arg[0].split(operation.value.capitalize())[1]
run_arg = None
if len(name_with_run_arg) > 1:
run_arg = name_with_run_arg[1]
return BenchmarkRow(
name_with_run_arg[0],
datastructure,
operation,
line["real_time"],
line["cpu_time"],
line["iterations"],
line["time_unit"],
run_arg,
)
def get_benchmark_res(args) -> Optional[GoogleBenchmarkResult]:
file_path = Path(args.log_file)
if not file_path.exists():
print("Error file {file_path} not found!")
return None
with file_path.open("r") as file:
data = json.load(file)
res = GoogleBenchmarkResult()
assert "benchmarks" in data, "There must be a benchmark list inside"
for benchmark in data["benchmarks"]:
res.add_result(get_row_data(benchmark))
return res
def plot_operation(results: GoogleBenchmarkResult, save: bool) -> None:
colors = ["red", "green", "blue", "yellow", "purple", "brown"]
assert results.operation is not None
fig = plt.figure()
for ds, benchmarks in results.datastructures.items():
if benchmarks:
# Print line chart
x_axis = [elem.real_time for elem in benchmarks]
y_axis = [elem.run_arg for elem in benchmarks]
plt.plot(x_axis, y_axis, marker="", color=colors.pop(0), linewidth="2", label=f"{ds}")
plt.title(f"Benchmark results for operation {results.operation.value}")
plt.xlabel(f"Time [{benchmarks[0].time_unit}]")
plt.grid(True)
plt.legend()
plt.draw()
else:
print(f"Nothing to do for {ds}...")
if save:
plt.savefig(f"{results.operation.value}.png")
plt.close(fig)
else:
plt.show()
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Process benchmark results.")
parser.add_argument("--log_file", type=str)
parser.add_argument("--save", type=bool, default=True)
return parser.parse_args()
def main():
args = parse_args()
res = get_benchmark_res(args)
if res is None:
print("Failed to get results from log file!")
sys.exit(1)
plot_operation(res, args.save)
if __name__ == "__main__":
main()

52
tools/plot/benchmark_datastructures.sh Executable file
View File

@ -0,0 +1,52 @@
#!/bin/bash
set -euox pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORKSPACE_DIR=${SCRIPT_DIR}/../../
CPUS=$(grep -c processor < /proc/cpuinfo)
# Get all benchmark files
BENCHMARK_FILES=$(find ${WORKSPACE_DIR}/tests/benchmark -type f -iname "data_structures_*")
function test_all() {
for BENCH_FILE in ${BENCHMARK_FILES[@]}; do
local BASE_NAME=$(basename $BENCH_FILE)
local NAME=${BASE_NAME%%.*}
echo "Running $NAME"
local TEST_FILE=${WORKSPACE_DIR}/build/tests/benchmark/${NAME}
if [[ -f "${TEST_FILE}" ]]; then
pushd ${WORKSPACE_DIR}/build
make -j${CPUS} memgraph__benchmark__${NAME}
popd
local JSON_OUTPUT=${NAME}_output.json
# Run benchmakr test
${WORKSPACE_DIR}/build/tests/benchmark/${NAME} --benchmark_format=json --benchmark_out=${JSON_OUTPUT}
# Run analyze script for benchmark test
python3 ${WORKSPACE_DIR}/tools/plot/benchmark_datastructures.py --log_file=${JSON_OUTPUT}
else
echo "File ${TEST_FILE} does not exist!"
fi
done
}
function test_memory() {
## We are testing only insert
local DATA_STRUCTURES=(SkipList StdMap StdSet BppTree)
for DATA_STRUCTURE in ${DATA_STRUCTURES[@]}; do
valgrind --tool=massif --massif-out-file=${DATA_STRUCTURE}.massif.out ${WORKSPACE_DIR}/build/tests/benchmark/data_structures_insert --benchmark_filter=BM_BenchmarkInsert${DATA_STRUCTURE}/10000 --benchmark_format=json --benchmark_out=${DATA_STRUCTURE}.json
done
}
ARG_1=${1:-"all"}
case ${ARG_1} in
all)
test_all
;;
memory)
test_memory
;;
*)
echo "Select either `all` or `memory` benchmark!"
;;
esac