tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix failing benchmark tests and implement cursor

Browse Source 
The benchmarking tests were failing because of the incorrect
implementation of the ScanAllByPrimaryKeyCursor. The previous
implementation caused the currently allocateable 1m edgeids to run out
very quickly, causing the the tests to freeze.
This commit is contained in:
gvolfing 2023-01-12 14:14:59 +01:00
parent 32ea124d4b
commit 41bb988fe9
10 changed files with 84 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/query/v2/context.hpp
View File

@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

2
src/query/v2/interpreter.cpp
View File

@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

40
src/query/v2/plan/operator.cpp
View File

@ -34,6 +34,7 @@
#include "query/v2/bindings/eval.hpp"
#include "query/v2/bindings/symbol_table.hpp"
#include "query/v2/context.hpp"
#include "query/v2/conversions.hpp"
#include "query/v2/db_accessor.hpp"
#include "query/v2/exceptions.hpp"
#include "query/v2/frontend/ast/ast.hpp"
@ -403,7 +404,7 @@ class DistributedScanAllAndFilterCursor : public Cursor {
if (label_.has_value()) {
request_label = request_router.LabelToName(*label_);
}
current_batch = request_router.ScanVertices(request_label);
current_batch = request_router.ScanVertices(request_label, std::nullopt);
}
current_vertex_it = current_batch.begin();
request_state_ = State::COMPLETED;
@ -468,13 +469,14 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor {
Symbol output_symbol, UniqueCursorPtr input_cursor, const char *op_name,
std::optional<storage::v3::LabelId> label,
std::optional<std::pair<storage::v3::PropertyId, Expression *>> property_expression_pair,
std::optional<std::vector<Expression *>> filter_expressions)
std::optional<std::vector<Expression *>> filter_expressions, std::optional<std::vector<Expression *>> primary_key)
: output_symbol_(output_symbol),
input_cursor_(std::move(input_cursor)),
op_name_(op_name),
label_(label),
property_expression_pair_(property_expression_pair),
filter_expressions_(filter_expressions) {
filter_expressions_(filter_expressions),
primary_key_(primary_key) {
ResetExecutionState();
}
@ -489,7 +491,32 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor {
if (label_.has_value()) {
request_label = request_router.LabelToName(*label_);
}
current_batch_ = request_router.ScanVertices(request_label);
current_batch_ = request_router.ScanVertices(request_label, std::nullopt);
}
current_vertex_it_ = current_batch_.begin();
request_state_ = State::COMPLETED;
return !current_batch_.empty();
}
bool MakeRequestSingleFrame(Frame &frame, RequestRouterInterface &request_router, ExecutionContext &context) {
{
SCOPED_REQUEST_WAIT_PROFILE;
std::optional<std::string> request_label = std::nullopt;
if (label_.has_value()) {
request_label = request_router.LabelToName(*label_);
}
// Evaluate the expressions that hold the PrimaryKey.
ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.request_router,
storage::v3::View::OLD);
std::vector<msgs::Value> pk;
MG_ASSERT(primary_key_);
for (auto primary_key : *primary_key_) {
pk.push_back(TypedValueToValue(primary_key->Accept(evaluator)));
}
current_batch_ = request_router.ScanVertices(request_label, pk);
}
current_vertex_it_ = current_batch_.begin();
request_state_ = State::COMPLETED;
@ -612,6 +639,7 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor {
std::optional<storage::v3::LabelId> label_;
std::optional<std::pair<storage::v3::PropertyId, Expression *>> property_expression_pair_;
std::optional<std::vector<Expression *>> filter_expressions_;
std::optional<std::vector<Expression *>> primary_key_;
std::optional<MultiFrame> own_multi_frames_;
std::optional<ValidFramesConsumer> valid_frames_consumer_;
ValidFramesConsumer::Iterator valid_frames_it_;
@ -727,8 +755,8 @@ UniqueCursorPtr ScanAllByPrimaryKey::MakeCursor(utils::MemoryResource *mem) cons
EventCounter::IncrementCounter(EventCounter::ScanAllByPrimaryKeyOperator);
return MakeUniqueCursorPtr<DistributedScanAllByPrimaryKeyCursor>(
mem, output_symbol_, input_->MakeCursor(mem), "ScanAll", std::nullopt /*label*/,
std::nullopt /*property_expression_pair*/, std::nullopt /*filter_expressions*/);
mem, output_symbol_, input_->MakeCursor(mem), "ScanAll", label_, std::nullopt /*property_expression_pair*/,
std::nullopt /*filter_expressions*/, primary_key_);
throw QueryRuntimeException("ScanAllByPrimaryKey cursur is yet to be implemented.");
}

37
src/query/v2/request_router.hpp
View File

@ -97,7 +97,8 @@ class RequestRouterInterface {
virtual void StartTransaction() = 0;
virtual void Commit() = 0;
virtual std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label) = 0;
virtual std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label,
std::optional<std::vector<msgs::Value>> primary_key) = 0;
virtual std::vector<msgs::CreateVerticesResponse> CreateVertices(std::vector<msgs::NewVertex> new_vertices) = 0;
virtual std::vector<msgs::ExpandOneResultRow> ExpandOne(msgs::ExpandOneRequest request) = 0;
virtual std::vector<msgs::CreateExpandResponse> CreateExpand(std::vector<msgs::NewExpand> new_edges) = 0;
@ -243,9 +244,16 @@ class RequestRouter : public RequestRouterInterface {
bool IsPrimaryLabel(storage::v3::LabelId label) const override { return shards_map_.label_spaces.contains(label); }
// TODO(kostasrim) Simplify return result
std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label) override {
std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label,
std::optional<std::vector<msgs::Value>> primary_key) override {
// create requests
std::vector<ShardRequestState<msgs::ScanVerticesRequest>> requests_to_be_sent = RequestsForScanVertices(label);
std::vector<ShardRequestState<msgs::ScanVerticesRequest>> requests_to_be_sent;
if (primary_key) {
requests_to_be_sent = RequestsForScanVertexByPrimaryKey(label, *primary_key);
} else {
requests_to_be_sent = RequestsForScanVertices(label);
}
spdlog::trace("created {} ScanVertices requests", requests_to_be_sent.size());
// begin all requests in parallel
@ -510,6 +518,29 @@ class RequestRouter : public RequestRouterInterface {
return requests;
}
std::vector<ShardRequestState<msgs::ScanVerticesRequest>> RequestsForScanVertexByPrimaryKey(
const std::optional<std::string> &label, const std::vector<msgs::Value> &primary_key) {
const auto label_id = shards_map_.GetLabelId(*label);
MG_ASSERT(label_id);
MG_ASSERT(IsPrimaryLabel(*label_id));
std::vector<ShardRequestState<msgs::ScanVerticesRequest>> requests = {};
auto pk_containing_shard =
shards_map_.GetShardForKey(*label, storage::conversions::ConvertPropertyVector(primary_key));
msgs::ScanVerticesRequest request;
request.transaction_id = transaction_id_;
request.batch_limit = 1;
request.start_id.second = primary_key;
ShardRequestState<msgs::ScanVerticesRequest> shard_request_state{
.shard = pk_containing_shard,
.request = std::move(request),
};
requests.emplace_back(std::move(shard_request_state));
return requests;
}
std::vector<ShardRequestState<msgs::ExpandOneRequest>> RequestsForExpandOne(const msgs::ExpandOneRequest &request) {
std::map<ShardMetadata, msgs::ExpandOneRequest> per_shard_request_table;
msgs::ExpandOneRequest top_level_rqst_template = request;

2
tests/mgbench/client.cpp
View File

@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

4
tests/simulation/request_router.cpp
View File

@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -154,7 +154,7 @@ void RunStorageRaft(Raft<IoImpl, MockedShardRsm, WriteRequests, WriteResponses,
}
void TestScanVertices(query::v2::RequestRouterInterface &request_router) {
auto result = request_router.ScanVertices("test_label");
auto result = request_router.ScanVertices("test_label", std::nullopt);
MG_ASSERT(result.size() == 2);
{
auto prop = result[0].GetProperty(msgs::PropertyId::FromUint(0));

4
tests/simulation/test_cluster.hpp
View File

@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -182,7 +182,7 @@ void ExecuteOp(query::v2::RequestRouter<SimulatorTransport> &request_router, std
void ExecuteOp(query::v2::RequestRouter<SimulatorTransport> &request_router, std::set<CompoundKey> &correctness_model,
ScanAll scan_all) {
auto results = request_router.ScanVertices("test_label");
auto results = request_router.ScanVertices("test_label", std::nullopt);
RC_ASSERT(results.size() == correctness_model.size());

4
tests/unit/high_density_shard_create_scan.cpp
View File

@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -192,7 +192,7 @@ void ExecuteOp(query::v2::RequestRouter<LocalTransport> &request_router, std::se
void ExecuteOp(query::v2::RequestRouter<LocalTransport> &request_router, std::set<CompoundKey> &correctness_model,
ScanAll scan_all) {
auto results = request_router.ScanVertices("test_label");
auto results = request_router.ScanVertices("test_label", std::nullopt);
spdlog::error("got {} results, model size is {}", results.size(), correctness_model.size());
EXPECT_EQ(results.size(), correctness_model.size());

4
tests/unit/machine_manager.cpp
View File

@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -111,7 +111,7 @@ ShardMap TestShardMap() {
template <typename RequestRouter>
void TestScanAll(RequestRouter &request_router) {
auto result = request_router.ScanVertices(kLabelName);
auto result = request_router.ScanVertices(kLabelName, std::nullopt);
EXPECT_EQ(result.size(), 2);
}

7
tests/unit/query_v2_expression_evaluator.cpp
View File

@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -84,7 +84,10 @@ class MockedRequestRouter : public RequestRouterInterface {
}
void StartTransaction() override {}
void Commit() override {}
std::vector<VertexAccessor> ScanVertices(std::optional<std::string> /* label */) override { return {}; }
std::vector<VertexAccessor> ScanVertices(std::optional<std::string> /* label */,
std::optional<std::vector<msgs::Value>> /*primary_key*/) override {
return {};
}
std::vector<CreateVerticesResponse> CreateVertices(
std::vector<memgraph::msgs::NewVertex> /* new_vertices */) override {