diff --git a/src/coordinator/shard_map.cpp b/src/coordinator/shard_map.cpp index ea167db87..f38e6f823 100644 --- a/src/coordinator/shard_map.cpp +++ b/src/coordinator/shard_map.cpp @@ -283,7 +283,7 @@ std::vector ShardMap::AssignShards(Address storage_manager, // TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info bool machine_contains_shard = false; - for (auto &aas : shard) { + for (auto &aas : shard.peers) { if (initialized.contains(aas.address.unique_id)) { machine_contains_shard = true; if (aas.status != Status::CONSENSUS_PARTICIPANT) { @@ -311,7 +311,7 @@ std::vector ShardMap::AssignShards(Address storage_manager, } } - if (!machine_contains_shard && shard.size() < label_space.replication_factor) { + if (!machine_contains_shard && shard.peers.size() < label_space.replication_factor) { // increment version for each new uuid for deterministic creation IncrementShardMapVersion(); @@ -337,7 +337,7 @@ std::vector ShardMap::AssignShards(Address storage_manager, .status = Status::INITIALIZING, }; - shard.emplace_back(aas); + shard.peers.emplace_back(aas); } } } @@ -360,9 +360,9 @@ bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, cons MG_ASSERT(!shards_in_map.contains(key)); MG_ASSERT(label_spaces.contains(label_id)); - // Finding the Shard that the new PrimaryKey should map to. + // Finding the ShardMetadata that the new PrimaryKey should map to. auto prev = std::prev(shards_in_map.upper_bound(key)); - Shard duplicated_shard = prev->second; + ShardMetadata duplicated_shard = prev->second; // Apply the split shards_in_map[key] = duplicated_shard; @@ -383,7 +383,7 @@ std::optional ShardMap::InitializeNewLabel(std::string label_name, std: labels.emplace(std::move(label_name), label_id); PrimaryKey initial_key = SchemaToMinKey(schema); - Shard empty_shard = {}; + ShardMetadata empty_shard = {}; Shards shards = { {initial_key, empty_shard}, @@ -479,7 +479,7 @@ Shards ShardMap::GetShardsForRange(const LabelName &label_name, const PrimaryKey return shards; } -Shard ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const { +ShardMetadata ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const { MG_ASSERT(labels.contains(label_name)); LabelId label_id = labels.at(label_name); @@ -492,7 +492,7 @@ Shard ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &ke return std::prev(label_space.shards.upper_bound(key))->second; } -Shard ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const { +ShardMetadata ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const { MG_ASSERT(label_spaces.contains(label_id)); const auto &label_space = label_spaces.at(label_id); @@ -556,12 +556,12 @@ EdgeTypeIdMap ShardMap::AllocateEdgeTypeIds(const std::vector &new bool ShardMap::ClusterInitialized() const { for (const auto &[label_id, label_space] : label_spaces) { for (const auto &[low_key, shard] : label_space.shards) { - if (shard.size() < label_space.replication_factor) { + if (shard.peers.size() < label_space.replication_factor) { spdlog::info("label_space below desired replication factor"); return false; } - for (const auto &aas : shard) { + for (const auto &aas : shard.peers) { if (aas.status != Status::CONSENSUS_PARTICIPANT) { spdlog::info("shard member not yet a CONSENSUS_PARTICIPANT"); return false; diff --git a/src/coordinator/shard_map.hpp b/src/coordinator/shard_map.hpp index 80c32eeba..fc408e965 100644 --- a/src/coordinator/shard_map.hpp +++ b/src/coordinator/shard_map.hpp @@ -76,8 +76,35 @@ struct AddressAndStatus { }; using PrimaryKey = std::vector; -using Shard = std::vector; -using Shards = std::map; + +struct ShardMetadata { + std::vector peers; + uint64_t version; + + friend std::ostream &operator<<(std::ostream &in, const ShardMetadata &shard) { + using utils::print_helpers::operator<<; + + in << "ShardMetadata { peers: "; + in << shard.peers; + in << " version: "; + in << shard.version; + in << " }"; + + return in; + } + + friend bool operator==(const ShardMetadata &lhs, const ShardMetadata &rhs) = default; + + friend bool operator<(const ShardMetadata &lhs, const ShardMetadata &rhs) { + if (lhs.peers != rhs.peers) { + return lhs.peers < rhs.peers; + } + + return lhs.version < rhs.version; + } +}; + +using Shards = std::map; using LabelName = std::string; using PropertyName = std::string; using EdgeTypeName = std::string; @@ -99,7 +126,7 @@ PrimaryKey SchemaToMinKey(const std::vector &schema); struct LabelSpace { std::vector schema; // Maps between the smallest primary key stored in the shard and the shard - std::map shards; + std::map shards; size_t replication_factor; friend std::ostream &operator<<(std::ostream &in, const LabelSpace &label_space) { @@ -160,9 +187,9 @@ struct ShardMap { Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const; - Shard GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const; + ShardMetadata GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const; - Shard GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const; + ShardMetadata GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const; PropertyMap AllocatePropertyIds(const std::vector &new_properties); diff --git a/src/query/v2/multiframe.cpp b/src/query/v2/multiframe.cpp index 88262e514..2cb591153 100644 --- a/src/query/v2/multiframe.cpp +++ b/src/query/v2/multiframe.cpp @@ -69,7 +69,7 @@ ValidFramesConsumer MultiFrame::GetValidFramesConsumer() { return ValidFramesCon InvalidFramesPopulator MultiFrame::GetInvalidFramesPopulator() { return InvalidFramesPopulator{*this}; } -ValidFramesReader::ValidFramesReader(MultiFrame &multiframe) : multiframe_(multiframe) { +ValidFramesReader::ValidFramesReader(MultiFrame &multiframe) : multiframe_(&multiframe) { /* From: https://en.cppreference.com/w/cpp/algorithm/find Returns an iterator to the first element in the range [first, last) that satisfies specific criteria: @@ -81,38 +81,55 @@ ValidFramesReader::ValidFramesReader(MultiFrame &multiframe) : multiframe_(multi */ auto it = std::find_if(multiframe.frames_.begin(), multiframe.frames_.end(), [](const auto &frame) { return !frame.IsValid(); }); - after_last_valid_frame_ = multiframe_.frames_.data() + std::distance(multiframe.frames_.begin(), it); + after_last_valid_frame_ = multiframe_->frames_.data() + std::distance(multiframe.frames_.begin(), it); +} + +ValidFramesReader::Iterator ValidFramesReader::begin() { + if (multiframe_->frames_[0].IsValid()) { + return Iterator{&multiframe_->frames_[0]}; + } + return end(); } -ValidFramesReader::Iterator ValidFramesReader::begin() { return Iterator{&multiframe_.frames_[0]}; } ValidFramesReader::Iterator ValidFramesReader::end() { return Iterator{after_last_valid_frame_}; } -ValidFramesModifier::ValidFramesModifier(MultiFrame &multiframe) : multiframe_(multiframe) {} +ValidFramesModifier::ValidFramesModifier(MultiFrame &multiframe) : multiframe_(&multiframe) {} -ValidFramesModifier::Iterator ValidFramesModifier::begin() { return Iterator{&multiframe_.frames_[0], *this}; } -ValidFramesModifier::Iterator ValidFramesModifier::end() { - return Iterator{multiframe_.frames_.data() + multiframe_.frames_.size(), *this}; +ValidFramesModifier::Iterator ValidFramesModifier::begin() { + if (multiframe_->frames_[0].IsValid()) { + return Iterator{&multiframe_->frames_[0], *this}; + } + return end(); } -ValidFramesConsumer::ValidFramesConsumer(MultiFrame &multiframe) : multiframe_(multiframe) {} +ValidFramesModifier::Iterator ValidFramesModifier::end() { + return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this}; +} + +ValidFramesConsumer::ValidFramesConsumer(MultiFrame &multiframe) : multiframe_(&multiframe) {} // NOLINTNEXTLINE (bugprone-exception-escape) ValidFramesConsumer::~ValidFramesConsumer() noexcept { // TODO Possible optimisation: only DefragmentValidFrames if one frame has been invalidated? Only if does not // cost too much to store it - multiframe_.DefragmentValidFrames(); + multiframe_->DefragmentValidFrames(); } -ValidFramesConsumer::Iterator ValidFramesConsumer::begin() { return Iterator{&multiframe_.frames_[0], *this}; } +ValidFramesConsumer::Iterator ValidFramesConsumer::begin() { + if (multiframe_->frames_[0].IsValid()) { + return Iterator{&multiframe_->frames_[0], *this}; + } + return end(); +} ValidFramesConsumer::Iterator ValidFramesConsumer::end() { - return Iterator{multiframe_.frames_.data() + multiframe_.frames_.size(), *this}; + return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this}; } -InvalidFramesPopulator::InvalidFramesPopulator(MultiFrame &multiframe) : multiframe_(multiframe) {} +InvalidFramesPopulator::InvalidFramesPopulator(MultiFrame &multiframe) : multiframe_(&multiframe) {} InvalidFramesPopulator::Iterator InvalidFramesPopulator::begin() { - for (auto &frame : multiframe_.frames_) { + for (auto &frame : multiframe_->frames_) { if (!frame.IsValid()) { return Iterator{&frame}; } @@ -121,7 +138,7 @@ InvalidFramesPopulator::Iterator InvalidFramesPopulator::begin() { } InvalidFramesPopulator::Iterator InvalidFramesPopulator::end() { - return Iterator{multiframe_.frames_.data() + multiframe_.frames_.size()}; + return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size()}; } } // namespace memgraph::query::v2 diff --git a/src/query/v2/multiframe.hpp b/src/query/v2/multiframe.hpp index 37e6fea11..e13eb07ac 100644 --- a/src/query/v2/multiframe.hpp +++ b/src/query/v2/multiframe.hpp @@ -137,7 +137,7 @@ class ValidFramesReader { private: FrameWithValidity *after_last_valid_frame_; - MultiFrame &multiframe_; + MultiFrame *multiframe_; }; class ValidFramesModifier { @@ -192,7 +192,7 @@ class ValidFramesModifier { Iterator end(); private: - MultiFrame &multiframe_; + MultiFrame *multiframe_; }; class ValidFramesConsumer { @@ -246,7 +246,7 @@ class ValidFramesConsumer { Iterator end(); private: - MultiFrame &multiframe_; + MultiFrame *multiframe_; }; class InvalidFramesPopulator { @@ -296,7 +296,7 @@ class InvalidFramesPopulator { Iterator end(); private: - MultiFrame &multiframe_; + MultiFrame *multiframe_; }; } // namespace memgraph::query::v2 diff --git a/src/query/v2/request_router.hpp b/src/query/v2/request_router.hpp index 116e884c2..3dd2f164b 100644 --- a/src/query/v2/request_router.hpp +++ b/src/query/v2/request_router.hpp @@ -50,7 +50,7 @@ template class RsmStorageClientManager { public: using CompoundKey = io::rsm::ShardRsmKey; - using Shard = coordinator::Shard; + using ShardMetadata = coordinator::ShardMetadata; RsmStorageClientManager() = default; RsmStorageClientManager(const RsmStorageClientManager &) = delete; RsmStorageClientManager(RsmStorageClientManager &&) = delete; @@ -58,25 +58,25 @@ class RsmStorageClientManager { RsmStorageClientManager &operator=(RsmStorageClientManager &&) = delete; ~RsmStorageClientManager() = default; - void AddClient(Shard key, TStorageClient client) { cli_cache_.emplace(std::move(key), std::move(client)); } + void AddClient(ShardMetadata key, TStorageClient client) { cli_cache_.emplace(std::move(key), std::move(client)); } - bool Exists(const Shard &key) { return cli_cache_.contains(key); } + bool Exists(const ShardMetadata &key) { return cli_cache_.contains(key); } void PurgeCache() { cli_cache_.clear(); } - TStorageClient &GetClient(const Shard &key) { + TStorageClient &GetClient(const ShardMetadata &key) { auto it = cli_cache_.find(key); MG_ASSERT(it != cli_cache_.end(), "Non-existing shard client"); return it->second; } private: - std::map cli_cache_; + std::map cli_cache_; }; template struct ShardRequestState { - memgraph::coordinator::Shard shard; + memgraph::coordinator::ShardMetadata shard; TRequest request; }; @@ -125,7 +125,7 @@ class RequestRouter : public RequestRouterInterface { using CoordinatorWriteRequests = coordinator::CoordinatorWriteRequests; using CoordinatorClient = coordinator::CoordinatorClient; using Address = io::Address; - using Shard = coordinator::Shard; + using ShardMetadata = coordinator::ShardMetadata; using ShardMap = coordinator::ShardMap; using CompoundKey = coordinator::PrimaryKey; using VertexAccessor = query::v2::accessors::VertexAccessor; @@ -403,7 +403,7 @@ class RequestRouter : public RequestRouterInterface { private: std::vector> RequestsForCreateVertices( const std::vector &new_vertices) { - std::map per_shard_request_table; + std::map per_shard_request_table; for (auto &new_vertex : new_vertices) { MG_ASSERT(!new_vertex.label_ids.empty(), "No label_ids provided for new vertex in RequestRouter::CreateVertices"); @@ -431,9 +431,9 @@ class RequestRouter : public RequestRouterInterface { std::vector> RequestsForCreateExpand( const std::vector &new_expands) { - std::map per_shard_request_table; + std::map per_shard_request_table; auto ensure_shard_exists_in_table = [&per_shard_request_table, - transaction_id = transaction_id_](const Shard &shard) { + transaction_id = transaction_id_](const ShardMetadata &shard) { if (!per_shard_request_table.contains(shard)) { msgs::CreateExpandRequest create_expand_request{.transaction_id = transaction_id}; per_shard_request_table.insert({shard, std::move(create_expand_request)}); @@ -484,7 +484,7 @@ class RequestRouter : public RequestRouterInterface { for (auto &shards : multi_shards) { for (auto &[key, shard] : shards) { - MG_ASSERT(!shard.empty()); + MG_ASSERT(!shard.peers.empty()); msgs::ScanVerticesRequest request; request.transaction_id = transaction_id_; @@ -503,7 +503,7 @@ class RequestRouter : public RequestRouterInterface { } std::vector> RequestsForExpandOne(const msgs::ExpandOneRequest &request) { - std::map per_shard_request_table; + std::map per_shard_request_table; msgs::ExpandOneRequest top_level_rqst_template = request; top_level_rqst_template.transaction_id = transaction_id_; top_level_rqst_template.src_vertices.clear(); @@ -533,7 +533,7 @@ class RequestRouter : public RequestRouterInterface { std::vector> RequestsForGetProperties( msgs::GetPropertiesRequest &&request) { - std::map per_shard_request_table; + std::map per_shard_request_table; auto top_level_rqst_template = request; top_level_rqst_template.transaction_id = transaction_id_; top_level_rqst_template.vertex_ids.clear(); @@ -571,7 +571,7 @@ class RequestRouter : public RequestRouterInterface { return requests; } - StorageClient &GetStorageClientForShard(Shard shard) { + StorageClient &GetStorageClientForShard(ShardMetadata shard) { if (!storage_cli_manager_.Exists(shard)) { AddStorageClientToManager(shard); } @@ -583,12 +583,12 @@ class RequestRouter : public RequestRouterInterface { return GetStorageClientForShard(std::move(shard)); } - void AddStorageClientToManager(Shard target_shard) { - MG_ASSERT(!target_shard.empty()); - auto leader_addr = target_shard.front(); + void AddStorageClientToManager(ShardMetadata target_shard) { + MG_ASSERT(!target_shard.peers.empty()); + auto leader_addr = target_shard.peers.front(); std::vector
addresses; - addresses.reserve(target_shard.size()); - for (auto &address : target_shard) { + addresses.reserve(target_shard.peers.size()); + for (auto &address : target_shard.peers) { addresses.push_back(std::move(address.address)); } auto cli = StorageClient(io_, std::move(leader_addr.address), std::move(addresses)); diff --git a/tests/benchmark/CMakeLists.txt b/tests/benchmark/CMakeLists.txt index 31f0eebc0..c7cededd8 100644 --- a/tests/benchmark/CMakeLists.txt +++ b/tests/benchmark/CMakeLists.txt @@ -9,11 +9,13 @@ function(add_benchmark test_cpp) get_filename_component(exec_name ${test_cpp} NAME_WE) set(target_name ${test_prefix}${exec_name}) add_executable(${target_name} ${test_cpp} ${ARGN}) + # OUTPUT_NAME sets the real name of a target when it is built and can be # used to help create two targets of the same name even though CMake # requires unique logical target names set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name}) target_link_libraries(${target_name} benchmark gflags) + # register test add_test(${target_name} ${exec_name}) add_dependencies(memgraph__benchmark ${target_name}) @@ -37,9 +39,9 @@ target_link_libraries(${test_prefix}profile mg-query) add_benchmark(query/stripped.cpp) target_link_libraries(${test_prefix}stripped mg-query) -if (MG_ENTERPRISE) -add_benchmark(rpc.cpp) -target_link_libraries(${test_prefix}rpc mg-rpc) +if(MG_ENTERPRISE) + add_benchmark(rpc.cpp) + target_link_libraries(${test_prefix}rpc mg-rpc) endif() add_benchmark(skip_list_random.cpp) @@ -65,3 +67,15 @@ target_link_libraries(${test_prefix}storage_v2_property_store mg-storage-v2) add_benchmark(future.cpp) target_link_libraries(${test_prefix}future mg-io) + +add_benchmark(data_structures_insert.cpp) +target_link_libraries(${test_prefix}data_structures_insert mg-utils mg-storage-v3) + +add_benchmark(data_structures_find.cpp) +target_link_libraries(${test_prefix}data_structures_find mg-utils mg-storage-v3) + +add_benchmark(data_structures_contains.cpp) +target_link_libraries(${test_prefix}data_structures_contains mg-utils mg-storage-v3) + +add_benchmark(data_structures_remove.cpp) +target_link_libraries(${test_prefix}data_structures_remove mg-utils mg-storage-v3) diff --git a/tests/benchmark/data_structures_common.hpp b/tests/benchmark/data_structures_common.hpp new file mode 100644 index 000000000..23fe394ee --- /dev/null +++ b/tests/benchmark/data_structures_common.hpp @@ -0,0 +1,58 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include + +#include "coordinator/hybrid_logical_clock.hpp" +#include "storage/v3/key_store.hpp" +#include "storage/v3/lexicographically_ordered_vertex.hpp" +#include "storage/v3/mvcc.hpp" +#include "storage/v3/transaction.hpp" +#include "utils/skip_list.hpp" + +namespace memgraph::benchmark { + +template +inline void PrepareData(utils::SkipList &skip_list, const int64_t num_elements) { + coordinator::Hlc start_timestamp; + storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION}; + for (auto i{0}; i < num_elements; ++i) { + auto acc = skip_list.access(); + acc.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{true}}}); + } +} + +template +inline void PrepareData(std::map &std_map, const int64_t num_elements) { + coordinator::Hlc start_timestamp; + storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION}; + auto *delta = storage::v3::CreateDeleteObjectDelta(&transaction); + for (auto i{0}; i < num_elements; ++i) { + std_map.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{i}}, + storage::v3::LexicographicallyOrderedVertex{storage::v3::Vertex{ + delta, std::vector{storage::v3::PropertyValue{true}}}}}); + } +} + +template +inline void PrepareData(std::set &std_set, const int64_t num_elements) { + coordinator::Hlc start_timestamp; + storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION}; + for (auto i{0}; i < num_elements; ++i) { + std_set.insert(std::vector{storage::v3::PropertyValue{true}}); + } +} + +} // namespace memgraph::benchmark diff --git a/tests/benchmark/data_structures_contains.cpp b/tests/benchmark/data_structures_contains.cpp new file mode 100644 index 000000000..47af24a1b --- /dev/null +++ b/tests/benchmark/data_structures_contains.cpp @@ -0,0 +1,105 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "data_structures_common.hpp" +#include "storage/v3/key_store.hpp" +#include "storage/v3/lexicographically_ordered_vertex.hpp" +#include "storage/v3/mvcc.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/transaction.hpp" +#include "storage/v3/vertex.hpp" +#include "utils/skip_list.hpp" + +namespace memgraph::benchmark { + +/////////////////////////////////////////////////////////////////////////////// +// Testing Contains Operation +/////////////////////////////////////////////////////////////////////////////// +static void BM_BenchmarkContainsSkipList(::benchmark::State &state) { + utils::SkipList skip_list; + PrepareData(skip_list, state.range(0)); + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t found_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + auto acc = skip_list.access(); + if (acc.contains(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}})) { + found_elems++; + } + } + } + state.SetItemsProcessed(found_elems); +} + +static void BM_BenchmarkContainsStdMap(::benchmark::State &state) { + std::map std_map; + PrepareData(std_map, state.range(0)); + + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t found_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + if (std_map.contains(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}})) { + found_elems++; + } + } + } + state.SetItemsProcessed(found_elems); +} + +static void BM_BenchmarkContainsStdSet(::benchmark::State &state) { + std::set std_set; + PrepareData(std_set, state.range(0)); + + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t found_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + if (std_set.contains(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}})) { + found_elems++; + } + } + } + state.SetItemsProcessed(found_elems); +} + +BENCHMARK(BM_BenchmarkContainsSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkContainsStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkContainsStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +} // namespace memgraph::benchmark + +BENCHMARK_MAIN(); diff --git a/tests/benchmark/data_structures_find.cpp b/tests/benchmark/data_structures_find.cpp new file mode 100644 index 000000000..d51771be5 --- /dev/null +++ b/tests/benchmark/data_structures_find.cpp @@ -0,0 +1,104 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "data_structures_common.hpp" +#include "storage/v3/key_store.hpp" +#include "storage/v3/lexicographically_ordered_vertex.hpp" +#include "storage/v3/mvcc.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/transaction.hpp" +#include "storage/v3/vertex.hpp" +#include "utils/skip_list.hpp" + +namespace memgraph::benchmark { + +/////////////////////////////////////////////////////////////////////////////// +// Testing Find Operation +/////////////////////////////////////////////////////////////////////////////// +static void BM_BenchmarkFindSkipList(::benchmark::State &state) { + utils::SkipList skip_list; + PrepareData(skip_list, state.range(0)); + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t found_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + auto acc = skip_list.access(); + if (acc.find(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}}) != acc.end()) { + found_elems++; + } + } + } + state.SetItemsProcessed(found_elems); +} + +static void BM_BenchmarkFindStdMap(::benchmark::State &state) { + std::map std_map; + PrepareData(std_map, state.range(0)); + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t found_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + if (std_map.find(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}}) != std_map.end()) { + found_elems++; + } + } + } + state.SetItemsProcessed(found_elems); +} + +static void BM_BenchmarkFindStdSet(::benchmark::State &state) { + std::set std_set; + PrepareData(std_set, state.range(0)); + + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t found_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + if (std_set.find(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}}) != std_set.end()) { + found_elems++; + } + } + } + state.SetItemsProcessed(found_elems); +} + +BENCHMARK(BM_BenchmarkFindSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkFindStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkFindStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +} // namespace memgraph::benchmark + +BENCHMARK_MAIN(); diff --git a/tests/benchmark/data_structures_insert.cpp b/tests/benchmark/data_structures_insert.cpp new file mode 100644 index 000000000..d83cbfe23 --- /dev/null +++ b/tests/benchmark/data_structures_insert.cpp @@ -0,0 +1,85 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "storage/v3/key_store.hpp" +#include "storage/v3/lexicographically_ordered_vertex.hpp" +#include "storage/v3/mvcc.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/transaction.hpp" +#include "storage/v3/vertex.hpp" +#include "utils/skip_list.hpp" + +namespace memgraph::benchmark { + +/////////////////////////////////////////////////////////////////////////////// +// Testing Insert Operation +/////////////////////////////////////////////////////////////////////////////// +static void BM_BenchmarkInsertSkipList(::benchmark::State &state) { + utils::SkipList skip_list; + coordinator::Hlc start_timestamp; + storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION}; + + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + auto acc = skip_list.access(); + acc.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{true}}}); + } + } +} + +static void BM_BenchmarkInsertStdMap(::benchmark::State &state) { + std::map std_map; + coordinator::Hlc start_timestamp; + storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION}; + auto *delta = storage::v3::CreateDeleteObjectDelta(&transaction); + + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + std_map.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{i}}, + storage::v3::LexicographicallyOrderedVertex{storage::v3::Vertex{ + delta, std::vector{storage::v3::PropertyValue{true}}}}}); + } + } +} + +static void BM_BenchmarkInsertStdSet(::benchmark::State &state) { + std::set std_set; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + std_set.insert( + storage::v3::PrimaryKey{std::vector{storage::v3::PropertyValue{true}}}); + } + } +} + +BENCHMARK(BM_BenchmarkInsertSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkInsertStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkInsertStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +} // namespace memgraph::benchmark + +BENCHMARK_MAIN(); diff --git a/tests/benchmark/data_structures_remove.cpp b/tests/benchmark/data_structures_remove.cpp new file mode 100644 index 000000000..641ad9453 --- /dev/null +++ b/tests/benchmark/data_structures_remove.cpp @@ -0,0 +1,106 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "data_structures_common.hpp" +#include "storage/v3/key_store.hpp" +#include "storage/v3/lexicographically_ordered_vertex.hpp" +#include "storage/v3/mvcc.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/transaction.hpp" +#include "storage/v3/vertex.hpp" +#include "utils/skip_list.hpp" + +namespace memgraph::benchmark { + +/////////////////////////////////////////////////////////////////////////////// +// Testing Remove Operation +/////////////////////////////////////////////////////////////////////////////// +static void BM_BenchmarkRemoveSkipList(::benchmark::State &state) { + utils::SkipList skip_list; + PrepareData(skip_list, state.range(0)); + + // So we can also have elements that don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t removed_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + auto acc = skip_list.access(); + if (acc.remove(storage::v3::PrimaryKey{storage::v3::PropertyValue(value)})) { + removed_elems++; + } + } + } + state.SetItemsProcessed(removed_elems); +} + +static void BM_BenchmarkRemoveStdMap(::benchmark::State &state) { + std::map std_map; + PrepareData(std_map, state.range(0)); + + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t removed_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + if (std_map.erase(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}}) > 0) { + removed_elems++; + } + } + } + state.SetItemsProcessed(removed_elems); +} + +static void BM_BenchmarkRemoveStdSet(::benchmark::State &state) { + std::set std_set; + PrepareData(std_set, state.range(0)); + + // So we can also have elements that does don't exist + std::mt19937 i_generator(std::random_device{}()); + std::uniform_int_distribution i_distribution(0, state.range(0) * 2); + int64_t removed_elems{0}; + for (auto _ : state) { + for (auto i{0}; i < state.range(0); ++i) { + int64_t value = i_distribution(i_generator); + if (std_set.erase(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}}) > 0) { + removed_elems++; + } + } + } + state.SetItemsProcessed(removed_elems); +} + +BENCHMARK(BM_BenchmarkRemoveSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkRemoveStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +BENCHMARK(BM_BenchmarkRemoveStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond); + +} // namespace memgraph::benchmark + +BENCHMARK_MAIN(); diff --git a/tests/benchmark/skip_list_common.hpp b/tests/benchmark/skip_list_common.hpp index 988a2dfb7..4b27281c8 100644 --- a/tests/benchmark/skip_list_common.hpp +++ b/tests/benchmark/skip_list_common.hpp @@ -1,4 +1,4 @@ -// Copyright 2021 Memgraph Ltd. +// Copyright 2022 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -11,11 +11,14 @@ #pragma once +#include #include #include +#include #include #include #include +#include #include #include @@ -26,7 +29,7 @@ DEFINE_int32(duration, 10, "Duration of test (in seconds)"); struct Stats { uint64_t total{0}; - uint64_t succ[4] = {0, 0, 0, 0}; + std::array succ = {0, 0, 0, 0}; }; const int OP_INSERT = 0; @@ -81,3 +84,27 @@ inline void RunConcurrentTest(std::function *, Stats *)> std::cout << "Total successful: " << tot << " (" << tot / FLAGS_duration << " calls/s)" << std::endl; std::cout << "Total ops: " << tops << " (" << tops / FLAGS_duration << " calls/s)" << std::endl; } + +inline void RunTest(std::function &, Stats &)> test_func) { + Stats stats; + std::atomic run{true}; + + { + std::jthread bg_thread(test_func, std::cref(run), std::ref(stats)); + std::this_thread::sleep_for(std::chrono::seconds(FLAGS_duration)); + run.store(false, std::memory_order_relaxed); + } + + std::cout << " Operations: " << stats.total << std::endl; + std::cout << " Successful insert: " << stats.succ[0] << std::endl; + std::cout << " Successful contains: " << stats.succ[1] << std::endl; + std::cout << " Successful remove: " << stats.succ[2] << std::endl; + std::cout << " Successful find: " << stats.succ[3] << std::endl; + std::cout << std::endl; + + const auto tot = std::accumulate(stats.succ.begin(), +stats.succ.begin() + 3, 0); + const auto tops = stats.total; + + std::cout << "Total successful: " << tot << " (" << tot / FLAGS_duration << " calls/s)" << std::endl; + std::cout << "Total ops: " << tops << " (" << tops / FLAGS_duration << " calls/s)" << std::endl; +} diff --git a/tests/simulation/request_router.cpp b/tests/simulation/request_router.cpp index 0f712793f..4248e7876 100644 --- a/tests/simulation/request_router.cpp +++ b/tests/simulation/request_router.cpp @@ -46,8 +46,8 @@ using coordinator::CoordinatorClient; using coordinator::CoordinatorRsm; using coordinator::HlcRequest; using coordinator::HlcResponse; -using coordinator::Shard; using coordinator::ShardMap; +using coordinator::ShardMetadata; using coordinator::Shards; using coordinator::Status; using io::Address; @@ -113,7 +113,7 @@ ShardMap CreateDummyShardmap(coordinator::Address a_io_1, coordinator::Address a AddressAndStatus aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT}; AddressAndStatus aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT}; - Shard shard1 = {aas1_1, aas1_2, aas1_3}; + ShardMetadata shard1 = ShardMetadata{.peers = {aas1_1, aas1_2, aas1_3}, .version = 1}; auto key1 = storage::v3::PropertyValue(0); auto key2 = storage::v3::PropertyValue(0); @@ -125,7 +125,7 @@ ShardMap CreateDummyShardmap(coordinator::Address a_io_1, coordinator::Address a AddressAndStatus aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT}; AddressAndStatus aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT}; - Shard shard2 = {aas2_1, aas2_2, aas2_3}; + ShardMetadata shard2 = ShardMetadata{.peers = {aas2_1, aas2_2, aas2_3}, .version = 1}; auto key3 = storage::v3::PropertyValue(12); auto key4 = storage::v3::PropertyValue(13); diff --git a/tests/simulation/sharded_map.cpp b/tests/simulation/sharded_map.cpp index 91661fc66..d27858abc 100644 --- a/tests/simulation/sharded_map.cpp +++ b/tests/simulation/sharded_map.cpp @@ -40,8 +40,8 @@ using memgraph::coordinator::CoordinatorRsm; using memgraph::coordinator::HlcRequest; using memgraph::coordinator::HlcResponse; using memgraph::coordinator::PrimaryKey; -using memgraph::coordinator::Shard; using memgraph::coordinator::ShardMap; +using memgraph::coordinator::ShardMetadata; using memgraph::coordinator::Shards; using memgraph::coordinator::Status; using memgraph::io::Address; @@ -109,7 +109,7 @@ ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Add AddressAndStatus aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT}; AddressAndStatus aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT}; - Shard shard1 = {aas1_1, aas1_2, aas1_3}; + ShardMetadata shard1 = ShardMetadata{.peers = {aas1_1, aas1_2, aas1_3}, .version = 1}; const auto key1 = PropertyValue(0); const auto key2 = PropertyValue(0); @@ -121,7 +121,7 @@ ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Add AddressAndStatus aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT}; AddressAndStatus aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT}; - Shard shard2 = {aas2_1, aas2_2, aas2_3}; + ShardMetadata shard2 = ShardMetadata{.peers = {aas2_1, aas2_2, aas2_3}, .version = 1}; auto key3 = PropertyValue(12); auto key4 = PropertyValue(13); @@ -131,10 +131,10 @@ ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Add return sm; } -std::optional DetermineShardLocation(const Shard &target_shard, const std::vector
&a_addrs, - ShardClient &a_client, const std::vector
&b_addrs, - ShardClient &b_client) { - for (const auto &addr : target_shard) { +std::optional DetermineShardLocation(const ShardMetadata &target_shard, + const std::vector
&a_addrs, ShardClient &a_client, + const std::vector
&b_addrs, ShardClient &b_client) { + for (const auto &addr : target_shard.peers) { if (addr.address == b_addrs[0]) { return &b_client; } @@ -275,7 +275,7 @@ int main() { const PrimaryKey compound_key = {cm_key_1, cm_key_2}; - // Look for Shard + // Look for ShardMetadata BasicResult read_res = coordinator_client.SendWriteRequest(req); diff --git a/tests/simulation/test_cluster.hpp b/tests/simulation/test_cluster.hpp index 3e14545a9..2e8bdf92f 100644 --- a/tests/simulation/test_cluster.hpp +++ b/tests/simulation/test_cluster.hpp @@ -47,8 +47,8 @@ using coordinator::GetShardMapRequest; using coordinator::GetShardMapResponse; using coordinator::Hlc; using coordinator::HlcResponse; -using coordinator::Shard; using coordinator::ShardMap; +using coordinator::ShardMetadata; using io::Address; using io::Io; using io::rsm::RsmClient; diff --git a/tests/unit/high_density_shard_create_scan.cpp b/tests/unit/high_density_shard_create_scan.cpp index cefa238ed..9fabf6ccc 100644 --- a/tests/unit/high_density_shard_create_scan.cpp +++ b/tests/unit/high_density_shard_create_scan.cpp @@ -44,8 +44,8 @@ using coordinator::GetShardMapRequest; using coordinator::GetShardMapResponse; using coordinator::Hlc; using coordinator::HlcResponse; -using coordinator::Shard; using coordinator::ShardMap; +using coordinator::ShardMetadata; using io::Address; using io::Io; using io::local_transport::LocalSystem; diff --git a/tests/unit/machine_manager.cpp b/tests/unit/machine_manager.cpp index 748233737..74b7d3863 100644 --- a/tests/unit/machine_manager.cpp +++ b/tests/unit/machine_manager.cpp @@ -45,8 +45,8 @@ using memgraph::coordinator::CoordinatorWriteRequests; using memgraph::coordinator::CoordinatorWriteResponses; using memgraph::coordinator::Hlc; using memgraph::coordinator::HlcResponse; -using memgraph::coordinator::Shard; using memgraph::coordinator::ShardMap; +using memgraph::coordinator::ShardMetadata; using memgraph::io::Io; using memgraph::io::local_transport::LocalSystem; using memgraph::io::local_transport::LocalTransport; diff --git a/tools/plot/benchmark_datastructures.py b/tools/plot/benchmark_datastructures.py new file mode 100644 index 000000000..f57516d4d --- /dev/null +++ b/tools/plot/benchmark_datastructures.py @@ -0,0 +1,185 @@ +# Copyright 2022 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +#################################### +# Benchmark datastructures analyzer +#################################### +# This scripts uses the output from dataset benchmark tests to plot charts +# comparing the results of different datastructures on the same operation. +# +# Note: Naming the tests is very important in order for this script to recognize +# which operation is being performed and on which DS, so it should come in this +# form: BM_Benchmark/ +# where run_argument will be added automatically by google benchmark framework + +import argparse +import json +import sys +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Any, Dict, List, Optional + +import matplotlib.pyplot as plt + + +class Operation(Enum): + CONTAINS = "contains" + FIND = "find" + INSERT = "insert" + RANDOM = "random" + REMOVE = "remove" + + @classmethod + def to_list(cls) -> List[str]: + return list(map(lambda c: c.value, cls)) + + @staticmethod + def get(s: str) -> Optional["Operation"]: + try: + return Operation[s.upper()] + except ValueError: + return None + + def __str__(self): + return str(self.value) + + +@dataclass(frozen=True) +class BenchmarkRow: + name: str + datastructure: str + operation: Operation + real_time: int + cpu_time: int + iterations: int + time_unit: str + run_arg: Optional[Any] + + +class GoogleBenchmarkResult: + def __init__(self): + self._operation = None + self._datastructures: Dict[str, List[BenchmarkRow]] = dict() + + def add_result(self, row: BenchmarkRow) -> None: + if self._operation is None: + self._operation = row.operation + assert self._operation is row.operation + if row.datastructure not in self._datastructures: + self._datastructures[row.datastructure] = [row] + else: + self._datastructures[row.datastructure].append(row) + + @property + def operation(self) -> Optional[Operation]: + return self._operation + + @property + def datastructures(self) -> Dict[str, List[BenchmarkRow]]: + return self._datastructures + + +def get_operation(s: str) -> Operation: + for op in Operation.to_list(): + if op.lower() in s.lower(): + operation_enum = Operation.get(op) + if operation_enum is not None: + return operation_enum + else: + print("Operation not found!") + sys.exit(1) + print("Operation not found!") + sys.exit(1) + + +def get_row_data(line: Dict[str, Any]) -> BenchmarkRow: + """ + Naming is very important, first must come an Operation name, and then a data + structure to test. + """ + full_name = line["name"].split("BM_Benchmark")[1] + name_with_run_arg = full_name.split("/") + operation = get_operation(name_with_run_arg[0]) + datastructure = name_with_run_arg[0].split(operation.value.capitalize())[1] + + run_arg = None + if len(name_with_run_arg) > 1: + run_arg = name_with_run_arg[1] + + return BenchmarkRow( + name_with_run_arg[0], + datastructure, + operation, + line["real_time"], + line["cpu_time"], + line["iterations"], + line["time_unit"], + run_arg, + ) + + +def get_benchmark_res(args) -> Optional[GoogleBenchmarkResult]: + file_path = Path(args.log_file) + if not file_path.exists(): + print("Error file {file_path} not found!") + return None + with file_path.open("r") as file: + data = json.load(file) + res = GoogleBenchmarkResult() + assert "benchmarks" in data, "There must be a benchmark list inside" + for benchmark in data["benchmarks"]: + res.add_result(get_row_data(benchmark)) + return res + + +def plot_operation(results: GoogleBenchmarkResult, save: bool) -> None: + colors = ["red", "green", "blue", "yellow", "purple", "brown"] + assert results.operation is not None + fig = plt.figure() + for ds, benchmarks in results.datastructures.items(): + if benchmarks: + # Print line chart + x_axis = [elem.real_time for elem in benchmarks] + y_axis = [elem.run_arg for elem in benchmarks] + plt.plot(x_axis, y_axis, marker="", color=colors.pop(0), linewidth="2", label=f"{ds}") + plt.title(f"Benchmark results for operation {results.operation.value}") + plt.xlabel(f"Time [{benchmarks[0].time_unit}]") + plt.grid(True) + plt.legend() + plt.draw() + else: + print(f"Nothing to do for {ds}...") + if save: + plt.savefig(f"{results.operation.value}.png") + plt.close(fig) + else: + plt.show() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Process benchmark results.") + parser.add_argument("--log_file", type=str) + parser.add_argument("--save", type=bool, default=True) + return parser.parse_args() + + +def main(): + args = parse_args() + res = get_benchmark_res(args) + if res is None: + print("Failed to get results from log file!") + sys.exit(1) + plot_operation(res, args.save) + + +if __name__ == "__main__": + main() diff --git a/tools/plot/benchmark_datastructures.sh b/tools/plot/benchmark_datastructures.sh new file mode 100755 index 000000000..2ab15be31 --- /dev/null +++ b/tools/plot/benchmark_datastructures.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -euox pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +WORKSPACE_DIR=${SCRIPT_DIR}/../../ +CPUS=$(grep -c processor < /proc/cpuinfo) + +# Get all benchmark files +BENCHMARK_FILES=$(find ${WORKSPACE_DIR}/tests/benchmark -type f -iname "data_structures_*") + +function test_all() { + for BENCH_FILE in ${BENCHMARK_FILES[@]}; do + local BASE_NAME=$(basename $BENCH_FILE) + local NAME=${BASE_NAME%%.*} + echo "Running $NAME" + local TEST_FILE=${WORKSPACE_DIR}/build/tests/benchmark/${NAME} + if [[ -f "${TEST_FILE}" ]]; then + pushd ${WORKSPACE_DIR}/build + make -j${CPUS} memgraph__benchmark__${NAME} + popd + local JSON_OUTPUT=${NAME}_output.json + # Run benchmakr test + ${WORKSPACE_DIR}/build/tests/benchmark/${NAME} --benchmark_format=json --benchmark_out=${JSON_OUTPUT} + # Run analyze script for benchmark test + python3 ${WORKSPACE_DIR}/tools/plot/benchmark_datastructures.py --log_file=${JSON_OUTPUT} + else + echo "File ${TEST_FILE} does not exist!" + fi + done +} + +function test_memory() { + ## We are testing only insert + local DATA_STRUCTURES=(SkipList StdMap StdSet BppTree) + for DATA_STRUCTURE in ${DATA_STRUCTURES[@]}; do + valgrind --tool=massif --massif-out-file=${DATA_STRUCTURE}.massif.out ${WORKSPACE_DIR}/build/tests/benchmark/data_structures_insert --benchmark_filter=BM_BenchmarkInsert${DATA_STRUCTURE}/10000 --benchmark_format=json --benchmark_out=${DATA_STRUCTURE}.json + done +} + +ARG_1=${1:-"all"} +case ${ARG_1} in + all) + test_all + ;; + memory) + test_memory + ;; + *) + echo "Select either `all` or `memory` benchmark!" + ;; +esac