From 03cf264b766127317b865a0c00e308595f1ea5cf Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Fri, 28 Oct 2022 08:29:27 +0000 Subject: [PATCH 01/13] Add test for 1k shards, 1k create vertices, and then a scan all --- tests/unit/1k_shards_1k_create_scanall.cpp | 258 +++++++++++++++++++++ tests/unit/CMakeLists.txt | 4 + 2 files changed, 262 insertions(+) create mode 100644 tests/unit/1k_shards_1k_create_scanall.cpp diff --git a/tests/unit/1k_shards_1k_create_scanall.cpp b/tests/unit/1k_shards_1k_create_scanall.cpp new file mode 100644 index 000000000..5fdbfafa8 --- /dev/null +++ b/tests/unit/1k_shards_1k_create_scanall.cpp @@ -0,0 +1,258 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include + +#include + +#include "coordinator/coordinator_client.hpp" +#include "coordinator/coordinator_rsm.hpp" +#include "coordinator/shard_map.hpp" +#include "io/address.hpp" +#include "io/local_transport/local_system.hpp" +#include "io/local_transport/local_transport.hpp" +#include "io/simulator/simulator.hpp" +#include "io/simulator/simulator_config.hpp" +#include "io/simulator/simulator_transport.hpp" +#include "machine_manager/machine_config.hpp" +#include "machine_manager/machine_manager.hpp" +#include "query/v2/requests.hpp" +#include "query/v2/shard_request_manager.hpp" +#include "utils/print_helpers.hpp" +#include "utils/variant_helpers.hpp" + +namespace memgraph::tests::simulation { + +using coordinator::Coordinator; +using coordinator::CoordinatorClient; +using coordinator::CoordinatorReadRequests; +using coordinator::CoordinatorWriteRequests; +using coordinator::CoordinatorWriteResponses; +using coordinator::GetShardMapRequest; +using coordinator::GetShardMapResponse; +using coordinator::Hlc; +using coordinator::HlcResponse; +using coordinator::Shard; +using coordinator::ShardMap; +using io::Address; +using io::Io; +using io::local_transport::LocalSystem; +using io::local_transport::LocalTransport; +using io::rsm::RsmClient; +using machine_manager::MachineConfig; +using machine_manager::MachineManager; +using msgs::ReadRequests; +using msgs::ReadResponses; +using msgs::WriteRequests; +using msgs::WriteResponses; +using storage::v3::LabelId; +using storage::v3::SchemaProperty; + +using CompoundKey = std::pair; +using ShardClient = RsmClient; + +struct CreateVertex { + int first; + int second; + + friend std::ostream &operator<<(std::ostream &in, const CreateVertex &add) { + in << "CreateVertex { first: " << add.first << ", second: " << add.second << " }"; + return in; + } +}; + +struct ScanAll { + friend std::ostream &operator<<(std::ostream &in, const ScanAll &get) { + in << "ScanAll {}"; + return in; + } +}; + +MachineManager MkMm(LocalSystem &local_system, std::vector
coordinator_addresses, Address addr, + ShardMap shard_map) { + MachineConfig config{ + .coordinator_addresses = coordinator_addresses, + .is_storage = true, + .is_coordinator = true, + .listen_ip = addr.last_known_ip, + .listen_port = addr.last_known_port, + }; + + Io io = local_system.Register(addr); + + Coordinator coordinator{shard_map}; + + return MachineManager{io, config, coordinator, shard_map}; +} + +void RunMachine(MachineManager mm) { mm.Run(); } + +void WaitForShardsToInitialize(CoordinatorClient &coordinator_client) { + // Call coordinator client's read method for GetShardMap and keep + // reading it until the shard map contains proper replicas for + // each shard in the label space. + + while (true) { + GetShardMapRequest req{}; + CoordinatorReadRequests read_req = req; + auto read_res = coordinator_client.SendReadRequest(read_req); + if (read_res.HasError()) { + // timed out + continue; + } + auto response_result = read_res.GetValue(); + auto response = std::get(response_result); + auto shard_map = response.shard_map; + + if (shard_map.ClusterInitialized()) { + spdlog::info("cluster stabilized - beginning workload"); + return; + } + } +} + +ShardMap TestShardMap(int n_splits, int replication_factor) { + ShardMap sm{}; + + const std::string label_name = std::string("test_label"); + + // register new properties + const std::vector property_names = {"property_1", "property_2"}; + const auto properties = sm.AllocatePropertyIds(property_names); + const auto property_id_1 = properties.at("property_1"); + const auto property_id_2 = properties.at("property_2"); + const auto type_1 = memgraph::common::SchemaType::INT; + const auto type_2 = memgraph::common::SchemaType::INT; + + // register new label space + std::vector schema = { + SchemaProperty{.property_id = property_id_1, .type = type_1}, + SchemaProperty{.property_id = property_id_2, .type = type_2}, + }; + + std::optional label_id = sm.InitializeNewLabel(label_name, schema, replication_factor, sm.shard_map_version); + MG_ASSERT(label_id.has_value()); + + // split the shard at N split points + for (int64_t i = 1; i < n_splits; ++i) { + const auto key1 = memgraph::storage::v3::PropertyValue(i); + const auto key2 = memgraph::storage::v3::PropertyValue(0); + + const auto split_point = {key1, key2}; + + const bool split_success = sm.SplitShard(sm.shard_map_version, label_id.value(), split_point); + + MG_ASSERT(split_success); + } + + return sm; +} + +void ExecuteOp(msgs::ShardRequestManager &shard_request_manager, + std::set &correctness_model, CreateVertex create_vertex) { + const auto key1 = memgraph::storage::v3::PropertyValue(create_vertex.first); + const auto key2 = memgraph::storage::v3::PropertyValue(create_vertex.second); + + std::vector primary_key = {msgs::Value(int64_t(create_vertex.first)), + msgs::Value(int64_t(create_vertex.second))}; + + if (correctness_model.contains(std::make_pair(create_vertex.first, create_vertex.second))) { + // TODO(tyler) remove this early-return when we have properly handled setting non-unique vertexes + return; + } + + msgs::ExecutionState state; + + auto label_id = shard_request_manager.NameToLabel("test_label"); + + msgs::NewVertex nv{.primary_key = primary_key}; + nv.label_ids.push_back({label_id}); + + std::vector new_vertices; + new_vertices.push_back(std::move(nv)); + + auto result = shard_request_manager.Request(state, std::move(new_vertices)); + + MG_ASSERT(result.size() == 1); + MG_ASSERT(result[0].success); + + correctness_model.emplace(std::make_pair(create_vertex.first, create_vertex.second)); +} + +void ExecuteOp(msgs::ShardRequestManager &shard_request_manager, + std::set &correctness_model, ScanAll scan_all) { + msgs::ExecutionState request{.label = "test_label"}; + + auto results = shard_request_manager.Request(request); + + MG_ASSERT(results.size() == correctness_model.size()); + + for (const auto &vertex_accessor : results) { + const auto properties = vertex_accessor.Properties(); + const auto primary_key = vertex_accessor.Id().second; + const CompoundKey model_key = std::make_pair(primary_key[0].int_v, primary_key[1].int_v); + MG_ASSERT(correctness_model.contains(model_key)); + } +} + +TEST(MachineManager, ManyShards) { + LocalSystem local_system; + + auto cli_addr = Address::TestAddress(1); + auto machine_1_addr = cli_addr.ForkUniqueAddress(); + + Io cli_io = local_system.Register(cli_addr); + Io cli_io_2 = local_system.Register(Address::TestAddress(2)); + + auto coordinator_addresses = std::vector{ + machine_1_addr, + }; + + auto shard_splits = 1024; + auto replication_factor = 1; + auto create_ops = 1000; + + ShardMap initialization_sm = TestShardMap(shard_splits, replication_factor); + + auto mm_1 = MkMm(local_system, coordinator_addresses, machine_1_addr, initialization_sm); + Address coordinator_address = mm_1.CoordinatorAddress(); + + auto mm_thread_1 = std::jthread(RunMachine, std::move(mm_1)); + + CoordinatorClient coordinator_client(cli_io, coordinator_address, {coordinator_address}); + WaitForShardsToInitialize(coordinator_client); + + msgs::ShardRequestManager shard_request_manager(std::move(coordinator_client), std::move(cli_io)); + + shard_request_manager.StartTransaction(); + + auto correctness_model = std::set{}; + + for (int i = 0; i < create_ops; i++) { + ExecuteOp(shard_request_manager, correctness_model, CreateVertex{.first = i, .second = i}); + } + + ExecuteOp(shard_request_manager, correctness_model, ScanAll{}); + + local_system.ShutDown(); + + auto histo = cli_io_2.ResponseLatencies(); + + using memgraph::utils::print_helpers::operator<<; + std::cout << "response latencies: " << histo << std::endl; +} + +} // namespace memgraph::tests::simulation diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index c8ed0d33a..bb723c1d3 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -442,3 +442,7 @@ target_link_libraries(${test_prefix}pretty_print_ast_to_original_expression_test # Tests for mg-coordinator add_unit_test(coordinator_shard_map.cpp) target_link_libraries(${test_prefix}coordinator_shard_map mg-coordinator) + +# Tests for mg-coordinator +add_unit_test(1k_shards_1k_create_scanall.cpp) +target_link_libraries(${test_prefix}1k_shards_1k_create_scanall mg-io mg-coordinator mg-storage-v3 mg-query-v2) From 6b0168cb3dbafe8549d21de06263f7d8e4856c3c Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Fri, 28 Oct 2022 08:51:12 +0000 Subject: [PATCH 02/13] Add LocalTransport::ResponseLatencies --- src/io/local_transport/local_transport.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/io/local_transport/local_transport.hpp b/src/io/local_transport/local_transport.hpp index 719605081..4fc6a4361 100644 --- a/src/io/local_transport/local_transport.hpp +++ b/src/io/local_transport/local_transport.hpp @@ -60,5 +60,9 @@ class LocalTransport { std::random_device rng; return distrib(rng); } + + std::unordered_map ResponseLatencies() { + return local_transport_handle_->ResponseLatencies(); + } }; }; // namespace memgraph::io::local_transport From 6834ce01fee25ce38673b0a74976ff4c958564d0 Mon Sep 17 00:00:00 2001 From: jbajic Date: Mon, 31 Oct 2022 16:26:56 +0100 Subject: [PATCH 03/13] Fix edge id allocator --- src/query/v2/context.hpp | 2 +- src/query/v2/interpreter.cpp | 2 +- src/query/v2/plan/operator.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/v2/context.hpp b/src/query/v2/context.hpp index 24b472438..338546f4d 100644 --- a/src/query/v2/context.hpp +++ b/src/query/v2/context.hpp @@ -98,7 +98,7 @@ struct ExecutionContext { ExecutionStats execution_stats; utils::AsyncTimer timer; msgs::ShardRequestManagerInterface *shard_request_manager{nullptr}; - IdAllocator edge_ids_alloc; + IdAllocator *edge_ids_alloc; }; static_assert(std::is_move_assignable_v, "ExecutionContext must be move assignable!"); diff --git a/src/query/v2/interpreter.cpp b/src/query/v2/interpreter.cpp index a1637b4c3..93ef20764 100644 --- a/src/query/v2/interpreter.cpp +++ b/src/query/v2/interpreter.cpp @@ -697,7 +697,7 @@ PullPlan::PullPlan(const std::shared_ptr plan, const Parameters &par ctx_.is_shutting_down = &interpreter_context->is_shutting_down; ctx_.is_profile_query = is_profile_query; ctx_.shard_request_manager = shard_request_manager; - ctx_.edge_ids_alloc = interpreter_context->edge_ids_alloc; + ctx_.edge_ids_alloc = &interpreter_context->edge_ids_alloc; } std::optional PullPlan::Pull(AnyStream *stream, std::optional n, diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 0832c60cf..1a83be36f 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -2402,7 +2402,7 @@ class DistributedCreateExpandCursor : public Cursor { std::vector ExpandCreationInfoToRequest(ExecutionContext &context, Frame &frame) const { std::vector edge_requests; for (const auto &edge_info : std::vector{self_.edge_info_}) { - msgs::NewExpand request{.id = {context.edge_ids_alloc.AllocateId()}}; + msgs::NewExpand request{.id = {context.edge_ids_alloc->AllocateId()}}; ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, nullptr, storage::v3::View::NEW); request.type = {edge_info.edge_type}; From 2f77eb96ffda7c562df094918c141c10646d664f Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Mon, 31 Oct 2022 16:49:46 +0000 Subject: [PATCH 04/13] Remove redundant ShardMap copy from MachineManager to avoid race conditions --- src/coordinator/shard_map.cpp | 19 +++++++++++++++++++ src/coordinator/shard_map.hpp | 3 +++ src/machine_manager/machine_manager.hpp | 5 ++--- src/memgraph.cpp | 3 +-- src/storage/v3/shard.cpp | 4 +++- src/storage/v3/shard.hpp | 3 ++- src/storage/v3/shard_manager.hpp | 21 ++++----------------- tests/simulation/test_cluster.hpp | 2 +- tests/unit/machine_manager.cpp | 2 +- 9 files changed, 36 insertions(+), 26 deletions(-) diff --git a/src/coordinator/shard_map.cpp b/src/coordinator/shard_map.cpp index 746f70368..9d7b8c0eb 100644 --- a/src/coordinator/shard_map.cpp +++ b/src/coordinator/shard_map.cpp @@ -228,6 +228,22 @@ Hlc ShardMap::IncrementShardMapVersion() noexcept { return shard_map_version; } +std::unordered_map ShardMap::IdToNames() { + std::unordered_map id_to_names; + + const auto map_type_ids = [&id_to_names](const auto &name_to_id_type) { + for (const auto &[name, id] : name_to_id_type) { + id_to_names.insert({id.AsUint(), name}); + } + }; + + map_type_ids(edge_types); + map_type_ids(labels); + map_type_ids(properties); + + return id_to_names; +} + Hlc ShardMap::GetHlc() const noexcept { return shard_map_version; } std::vector ShardMap::AssignShards(Address storage_manager, @@ -259,6 +275,7 @@ std::vector ShardMap::AssignShards(Address storage_manager, if (same_machine) { machine_contains_shard = true; spdlog::info("reminding shard manager that they should begin participating in shard"); + ret.push_back(ShardToInitialize{ .uuid = aas.address.unique_id, .label_id = label_id, @@ -266,6 +283,7 @@ std::vector ShardMap::AssignShards(Address storage_manager, .max_key = high_key, .schema = schemas[label_id], .config = Config{}, + .id_to_names = IdToNames(), }); } } @@ -286,6 +304,7 @@ std::vector ShardMap::AssignShards(Address storage_manager, .max_key = high_key, .schema = schemas[label_id], .config = Config{}, + .id_to_names = IdToNames(), }); AddressAndStatus aas = { diff --git a/src/coordinator/shard_map.hpp b/src/coordinator/shard_map.hpp index d8b3e0f6a..80c32eeba 100644 --- a/src/coordinator/shard_map.hpp +++ b/src/coordinator/shard_map.hpp @@ -91,6 +91,7 @@ struct ShardToInitialize { std::optional max_key; std::vector schema; Config config; + std::unordered_map id_to_names; }; PrimaryKey SchemaToMinKey(const std::vector &schema); @@ -137,6 +138,8 @@ struct ShardMap { Hlc IncrementShardMapVersion() noexcept; Hlc GetHlc() const noexcept; + std::unordered_map IdToNames(); + // Returns the shard UUIDs that have been assigned but not yet acknowledged for this storage manager std::vector AssignShards(Address storage_manager, std::set initialized); diff --git a/src/machine_manager/machine_manager.hpp b/src/machine_manager/machine_manager.hpp index aab658429..fc4d903a4 100644 --- a/src/machine_manager/machine_manager.hpp +++ b/src/machine_manager/machine_manager.hpp @@ -18,7 +18,6 @@ #include #include #include -#include "coordinator/shard_map.hpp" namespace memgraph::machine_manager { @@ -70,11 +69,11 @@ class MachineManager { public: // TODO initialize ShardManager with "real" coordinator addresses instead of io.GetAddress // which is only true for single-machine config. - MachineManager(io::Io io, MachineConfig config, Coordinator coordinator, coordinator::ShardMap &shard_map) + MachineManager(io::Io io, MachineConfig config, Coordinator coordinator) : io_(io), config_(config), coordinator_{std::move(io.ForkLocal()), {}, std::move(coordinator)}, - shard_manager_{io.ForkLocal(), coordinator_.GetAddress(), shard_map} {} + shard_manager_{io.ForkLocal(), coordinator_.GetAddress()} {} Address CoordinatorAddress() { return coordinator_.GetAddress(); } diff --git a/src/memgraph.cpp b/src/memgraph.cpp index 301b3ed36..0f80266e5 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -658,8 +658,7 @@ int main(int argc, char **argv) { memgraph::coordinator::Coordinator coordinator{sm}; - memgraph::machine_manager::MachineManager mm{io, config, coordinator, - sm}; + memgraph::machine_manager::MachineManager mm{io, config, coordinator}; std::jthread mm_thread([&mm] { mm.Run(); }); memgraph::query::v2::InterpreterContext interpreter_context{ diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index ebc571333..d6ba56ac3 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -321,7 +321,8 @@ bool VerticesIterable::Iterator::operator==(const Iterator &other) const { } Shard::Shard(const LabelId primary_label, const PrimaryKey min_primary_key, - const std::optional max_primary_key, std::vector schema, Config config) + const std::optional max_primary_key, std::vector schema, Config config, + std::unordered_map id_to_name) : primary_label_{primary_label}, min_primary_key_{min_primary_key}, max_primary_key_{max_primary_key}, @@ -334,6 +335,7 @@ Shard::Shard(const LabelId primary_label, const PrimaryKey min_primary_key, epoch_id_{utils::GenerateUUID()}, global_locker_{file_retainer_.AddLocker()} { CreateSchema(primary_label_, schema); + StoreMapping(std::move(id_to_name)); } Shard::~Shard() {} diff --git a/src/storage/v3/shard.hpp b/src/storage/v3/shard.hpp index 268d350ae..a3f1fc90d 100644 --- a/src/storage/v3/shard.hpp +++ b/src/storage/v3/shard.hpp @@ -189,7 +189,8 @@ class Shard final { /// @throw std::system_error /// @throw std::bad_alloc explicit Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional max_primary_key, - std::vector schema, Config config = Config()); + std::vector schema, Config config = Config(), + std::unordered_map id_to_name = {}); Shard(const Shard &) = delete; Shard(Shard &&) noexcept = delete; diff --git a/src/storage/v3/shard_manager.hpp b/src/storage/v3/shard_manager.hpp index a119148e9..b379bf616 100644 --- a/src/storage/v3/shard_manager.hpp +++ b/src/storage/v3/shard_manager.hpp @@ -77,8 +77,7 @@ static_assert(kMinimumCronInterval < kMaximumCronInterval, template class ShardManager { public: - ShardManager(io::Io io, Address coordinator_leader, coordinator::ShardMap shard_map) - : io_(io), coordinator_leader_(coordinator_leader), shard_map_{std::move(shard_map)} {} + ShardManager(io::Io io, Address coordinator_leader) : io_(io), coordinator_leader_(coordinator_leader) {} /// Periodic protocol maintenance. Returns the time that Cron should be called again /// in the future. @@ -137,7 +136,6 @@ class ShardManager { std::priority_queue, std::vector>, std::greater<>> cron_schedule_; Time next_cron_ = Time::min(); Address coordinator_leader_; - coordinator::ShardMap shard_map_; std::optional>> heartbeat_res_; // TODO(tyler) over time remove items from initialized_but_not_confirmed_rsm_ @@ -210,22 +208,11 @@ class ShardManager { io_addr.unique_id = to_init.uuid; rsm_io.SetAddress(io_addr); - // TODO(tyler) get geers from Coordinator in HeartbeatResponse + // TODO(tyler) get peers from Coordinator in HeartbeatResponse std::vector
rsm_peers = {}; - std::unique_ptr shard = - std::make_unique(to_init.label_id, to_init.min_key, to_init.max_key, to_init.schema, to_init.config); - // TODO(jbajic) Should be sync with coordinator and not passed - std::unordered_map id_to_name; - const auto map_type_ids = [&id_to_name](const auto &name_to_id_type) { - for (const auto &[name, id] : name_to_id_type) { - id_to_name.insert({id.AsUint(), name}); - } - }; - map_type_ids(shard_map_.edge_types); - map_type_ids(shard_map_.labels); - map_type_ids(shard_map_.properties); - shard->StoreMapping(std::move(id_to_name)); + std::unique_ptr shard = std::make_unique(to_init.label_id, to_init.min_key, to_init.max_key, + to_init.schema, to_init.config, to_init.id_to_names); ShardRsm rsm_state{std::move(shard)}; diff --git a/tests/simulation/test_cluster.hpp b/tests/simulation/test_cluster.hpp index d9870bfff..b87bda998 100644 --- a/tests/simulation/test_cluster.hpp +++ b/tests/simulation/test_cluster.hpp @@ -81,7 +81,7 @@ MachineManager MkMm(Simulator &simulator, std::vector mm) { mm.Run(); } diff --git a/tests/unit/machine_manager.cpp b/tests/unit/machine_manager.cpp index 99e18a961..1d69da5c5 100644 --- a/tests/unit/machine_manager.cpp +++ b/tests/unit/machine_manager.cpp @@ -185,7 +185,7 @@ MachineManager MkMm(LocalSystem &local_system, std::vector mm) { mm.Run(); } From fa972813d290fcfb1b43fc5b5de61a8e76474f92 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Mon, 31 Oct 2022 19:05:19 +0100 Subject: [PATCH 05/13] Update src/coordinator/shard_map.cpp Co-authored-by: Jure Bajic --- src/coordinator/shard_map.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coordinator/shard_map.cpp b/src/coordinator/shard_map.cpp index 9d7b8c0eb..bcbc08f57 100644 --- a/src/coordinator/shard_map.cpp +++ b/src/coordinator/shard_map.cpp @@ -233,7 +233,7 @@ std::unordered_map ShardMap::IdToNames() { const auto map_type_ids = [&id_to_names](const auto &name_to_id_type) { for (const auto &[name, id] : name_to_id_type) { - id_to_names.insert({id.AsUint(), name}); + id_to_names.emplace(id.AsUint(), name); } }; From 119da2d7a767244d54ef6520c023695d6329fdae Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 1 Nov 2022 09:51:43 +0100 Subject: [PATCH 06/13] Update CMakeLists.txt --- tests/unit/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index bb723c1d3..52e3a7eef 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -443,6 +443,6 @@ target_link_libraries(${test_prefix}pretty_print_ast_to_original_expression_test add_unit_test(coordinator_shard_map.cpp) target_link_libraries(${test_prefix}coordinator_shard_map mg-coordinator) -# Tests for mg-coordinator +# Tests for 1000 shards, 1000 creates, scan add_unit_test(1k_shards_1k_create_scanall.cpp) target_link_libraries(${test_prefix}1k_shards_1k_create_scanall mg-io mg-coordinator mg-storage-v3 mg-query-v2) From 77c2afc9e7bde3e4433ce98a9386f351208be243 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 1 Nov 2022 15:27:35 +0000 Subject: [PATCH 07/13] Add TODO about ID mapping improvements --- src/coordinator/shard_map.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coordinator/shard_map.cpp b/src/coordinator/shard_map.cpp index bcbc08f57..8e38bec6c 100644 --- a/src/coordinator/shard_map.cpp +++ b/src/coordinator/shard_map.cpp @@ -228,6 +228,7 @@ Hlc ShardMap::IncrementShardMapVersion() noexcept { return shard_map_version; } +// TODO(antaljanosbenjamin) use a single map for all name id mapping and a single counter to maintain the next id std::unordered_map ShardMap::IdToNames() { std::unordered_map id_to_names; From 5674ef40160a13bd7bc9732ae73865dd599a1d06 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 1 Nov 2022 16:30:37 +0100 Subject: [PATCH 08/13] Update tests/unit/1k_shards_1k_create_scanall.cpp Co-authored-by: Kostas Kyrimis --- tests/unit/1k_shards_1k_create_scanall.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/1k_shards_1k_create_scanall.cpp b/tests/unit/1k_shards_1k_create_scanall.cpp index 5fdbfafa8..7bfbbc27b 100644 --- a/tests/unit/1k_shards_1k_create_scanall.cpp +++ b/tests/unit/1k_shards_1k_create_scanall.cpp @@ -84,7 +84,7 @@ struct ScanAll { MachineManager MkMm(LocalSystem &local_system, std::vector
coordinator_addresses, Address addr, ShardMap shard_map) { MachineConfig config{ - .coordinator_addresses = coordinator_addresses, + .coordinator_addresses = std::move(coordinator_addresses), .is_storage = true, .is_coordinator = true, .listen_ip = addr.last_known_ip, From c55ca836a4ac4a6a840e6e2994c30ff7b5d96a17 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 1 Nov 2022 16:31:06 +0100 Subject: [PATCH 09/13] Update tests/unit/1k_shards_1k_create_scanall.cpp Co-authored-by: Kostas Kyrimis --- tests/unit/1k_shards_1k_create_scanall.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/1k_shards_1k_create_scanall.cpp b/tests/unit/1k_shards_1k_create_scanall.cpp index 7bfbbc27b..402d87cc4 100644 --- a/tests/unit/1k_shards_1k_create_scanall.cpp +++ b/tests/unit/1k_shards_1k_create_scanall.cpp @@ -95,7 +95,7 @@ MachineManager MkMm(LocalSystem &local_system, std::vector mm) { mm.Run(); } From 9a1258a708a6eccc2e974b81fc92ec6802d9901c Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 1 Nov 2022 16:31:14 +0100 Subject: [PATCH 10/13] Update tests/unit/1k_shards_1k_create_scanall.cpp Co-authored-by: Kostas Kyrimis --- tests/unit/1k_shards_1k_create_scanall.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/1k_shards_1k_create_scanall.cpp b/tests/unit/1k_shards_1k_create_scanall.cpp index 402d87cc4..603dc6a2f 100644 --- a/tests/unit/1k_shards_1k_create_scanall.cpp +++ b/tests/unit/1k_shards_1k_create_scanall.cpp @@ -127,7 +127,7 @@ void WaitForShardsToInitialize(CoordinatorClient &coordinator_cl ShardMap TestShardMap(int n_splits, int replication_factor) { ShardMap sm{}; - const std::string label_name = std::string("test_label"); + const auto label_name = std::string("test_label"); // register new properties const std::vector property_names = {"property_1", "property_2"}; From 6dd57426f80a49ec3bdb965e42f2a280a6187171 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 1 Nov 2022 15:51:37 +0000 Subject: [PATCH 11/13] Revert broken code suggestion --- tests/unit/1k_shards_1k_create_scanall.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/1k_shards_1k_create_scanall.cpp b/tests/unit/1k_shards_1k_create_scanall.cpp index 603dc6a2f..468a0e8b0 100644 --- a/tests/unit/1k_shards_1k_create_scanall.cpp +++ b/tests/unit/1k_shards_1k_create_scanall.cpp @@ -95,7 +95,7 @@ MachineManager MkMm(LocalSystem &local_system, std::vector mm) { mm.Run(); } From 599802033f33f00a1078ccf608049c6fe070a528 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 1 Nov 2022 17:21:15 +0100 Subject: [PATCH 12/13] Reformat TODO comment --- src/coordinator/shard_map.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coordinator/shard_map.cpp b/src/coordinator/shard_map.cpp index 8e38bec6c..d0b2cf9ad 100644 --- a/src/coordinator/shard_map.cpp +++ b/src/coordinator/shard_map.cpp @@ -228,7 +228,8 @@ Hlc ShardMap::IncrementShardMapVersion() noexcept { return shard_map_version; } -// TODO(antaljanosbenjamin) use a single map for all name id mapping and a single counter to maintain the next id +// TODO(antaljanosbenjamin) use a single map for all name id +// mapping and a single counter to maintain the next id std::unordered_map ShardMap::IdToNames() { std::unordered_map id_to_names; From 84509fa4773420ba40ed3428136f7353eae5e42b Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 1 Nov 2022 17:07:14 +0000 Subject: [PATCH 13/13] Fix merge conflict with MachineManager constructor --- tests/unit/1k_shards_1k_create_scanall.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/1k_shards_1k_create_scanall.cpp b/tests/unit/1k_shards_1k_create_scanall.cpp index 468a0e8b0..eb1f8a5d5 100644 --- a/tests/unit/1k_shards_1k_create_scanall.cpp +++ b/tests/unit/1k_shards_1k_create_scanall.cpp @@ -95,7 +95,7 @@ MachineManager MkMm(LocalSystem &local_system, std::vector mm) { mm.Run(); }