Compare commits

...

5 Commits

Author SHA1 Message Date
DavIvek
8dbf8472f5 minor fix 2024-03-11 13:21:31 +01:00
DavIvek
03f8910bdd refactor merge vertices 2024-03-11 10:46:09 +01:00
DavIvek
39b3ea136d
Merge branch 'master' into fix-merging-vertices-on-disk 2024-03-11 08:59:22 +01:00
DavIvek
80e1e2d9ca fix mg_assert failing 2024-03-08 22:55:45 +01:00
DavIvek
077f9e6e99 don't create new delta when merging caches 2024-03-08 17:00:28 +01:00
3 changed files with 24 additions and 26 deletions

View File

@ -51,6 +51,7 @@
#include "storage/v2/storage.hpp"
#include "storage/v2/storage_error.hpp"
#include "storage/v2/transaction.hpp"
#include "storage/v2/vertex.hpp"
#include "storage/v2/vertex_accessor.hpp"
#include "storage/v2/vertices_iterable.hpp"
#include "storage/v2/view.hpp"
@ -516,8 +517,8 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, View view) {
transaction_.index_deltas_storage_.emplace_back();
auto &index_deltas = transaction_.index_deltas_storage_.back();
auto gids = disk_storage->MergeVerticesFromMainCacheWithLabelIndexCache(&transaction_, label, view, index_deltas,
indexed_vertices.get());
auto gids =
disk_storage->MergeVerticesFromMainCacheWithLabelIndexCache(&transaction_, label, view, indexed_vertices.get());
disk_storage->LoadVerticesFromDiskLabelIndex(&transaction_, label, gids, index_deltas, indexed_vertices.get());
return VerticesIterable(AllVerticesIterable(indexed_vertices->access(), storage_, &transaction_, view));
@ -544,7 +545,7 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId p
};
const auto gids = disk_storage->MergeVerticesFromMainCacheWithLabelPropertyIndexCache(
&transaction_, label, property, view, index_deltas, indexed_vertices.get(), label_property_filter);
&transaction_, label, property, view, indexed_vertices.get(), label_property_filter);
const auto disk_label_property_filter = [](const std::string &key, const std::string &label_property_prefix,
const std::unordered_set<Gid> &gids, Gid curr_gid) -> bool {
@ -580,7 +581,7 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId p
};
const auto gids = disk_storage->MergeVerticesFromMainCacheWithLabelPropertyIndexCache(
&transaction_, label, property, view, index_deltas, indexed_vertices.get(), label_property_filter);
&transaction_, label, property, view, indexed_vertices.get(), label_property_filter);
disk_storage->LoadVerticesFromDiskLabelPropertyIndexWithPointValueLookup(&transaction_, label, property, gids, value,
index_deltas, indexed_vertices.get());
@ -606,7 +607,7 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId p
auto &index_deltas = transaction_.index_deltas_storage_.back();
const auto gids = disk_storage->MergeVerticesFromMainCacheWithLabelPropertyIndexCacheForIntervalSearch(
&transaction_, label, property, view, lower_bound, upper_bound, index_deltas, indexed_vertices.get());
&transaction_, label, property, view, lower_bound, upper_bound, indexed_vertices.get());
disk_storage->LoadVerticesFromDiskLabelPropertyIndexForIntervalSearch(
&transaction_, label, property, gids, lower_bound, upper_bound, index_deltas, indexed_vertices.get());
@ -616,8 +617,7 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId p
/// TODO: (andi) This should probably go into some other class not the storage. All utils methods
std::unordered_set<Gid> DiskStorage::MergeVerticesFromMainCacheWithLabelIndexCache(
Transaction *transaction, LabelId label, View view, std::list<Delta> &index_deltas,
utils::SkipList<Vertex> *indexed_vertices) {
Transaction *transaction, LabelId label, View view, utils::SkipList<Vertex> *indexed_vertices) {
auto main_cache_acc = transaction->vertices_->access();
std::unordered_set<Gid> gids;
gids.reserve(main_cache_acc.size());
@ -626,12 +626,10 @@ std::unordered_set<Gid> DiskStorage::MergeVerticesFromMainCacheWithLabelIndexCac
gids.insert(vertex.gid);
if (VertexHasLabel(vertex, label, transaction, view)) {
spdlog::trace("Loaded vertex with gid: {} from main index storage to label index", vertex.gid.ToString());
uint64_t ts = utils::GetEarliestTimestamp(vertex.delta);
/// TODO: here are doing serialization and then later deserialization again -> expensive
LoadVertexToLabelIndexCache(transaction, utils::SerializeVertexAsKeyForLabelIndex(label, vertex.gid),
utils::SerializeVertexAsValueForLabelIndex(label, vertex.labels, vertex.properties),
CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::nullopt, ts),
indexed_vertices->access());
vertex.delta, indexed_vertices->access());
}
}
return gids;
@ -668,8 +666,8 @@ void DiskStorage::LoadVerticesFromDiskLabelIndex(Transaction *transaction, Label
}
std::unordered_set<Gid> DiskStorage::MergeVerticesFromMainCacheWithLabelPropertyIndexCache(
Transaction *transaction, LabelId label, PropertyId property, View view, std::list<Delta> &index_deltas,
utils::SkipList<Vertex> *indexed_vertices, const auto &label_property_filter) {
Transaction *transaction, LabelId label, PropertyId property, View view, utils::SkipList<Vertex> *indexed_vertices,
const auto &label_property_filter) {
auto main_cache_acc = transaction->vertices_->access();
std::unordered_set<storage::Gid> gids;
gids.reserve(main_cache_acc.size());
@ -677,11 +675,10 @@ std::unordered_set<Gid> DiskStorage::MergeVerticesFromMainCacheWithLabelProperty
for (const auto &vertex : main_cache_acc) {
gids.insert(vertex.gid);
if (label_property_filter(vertex, label, property, view)) {
uint64_t ts = utils::GetEarliestTimestamp(vertex.delta);
LoadVertexToLabelPropertyIndexCache(
transaction, utils::SerializeVertexAsKeyForLabelPropertyIndex(label, property, vertex.gid),
utils::SerializeVertexAsValueForLabelPropertyIndex(label, vertex.labels, vertex.properties),
CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::nullopt, ts), indexed_vertices->access());
utils::SerializeVertexAsValueForLabelPropertyIndex(label, vertex.labels, vertex.properties), vertex.delta,
indexed_vertices->access());
}
}
@ -751,8 +748,7 @@ void DiskStorage::LoadVerticesFromDiskLabelPropertyIndexWithPointValueLookup(
std::unordered_set<Gid> DiskStorage::MergeVerticesFromMainCacheWithLabelPropertyIndexCacheForIntervalSearch(
Transaction *transaction, LabelId label, PropertyId property, View view,
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
const std::optional<utils::Bound<PropertyValue>> &upper_bound, std::list<Delta> &index_deltas,
utils::SkipList<Vertex> *indexed_vertices) {
const std::optional<utils::Bound<PropertyValue>> &upper_bound, utils::SkipList<Vertex> *indexed_vertices) {
auto main_cache_acc = transaction->vertices_->access();
std::unordered_set<storage::Gid> gids;
gids.reserve(main_cache_acc.size());
@ -762,11 +758,10 @@ std::unordered_set<Gid> DiskStorage::MergeVerticesFromMainCacheWithLabelProperty
auto prop_value = GetVertexProperty(vertex, property, transaction, view);
if (VertexHasLabel(vertex, label, transaction, view) &&
IsPropertyValueWithinInterval(prop_value, lower_bound, upper_bound)) {
uint64_t ts = utils::GetEarliestTimestamp(vertex.delta);
LoadVertexToLabelPropertyIndexCache(
transaction, utils::SerializeVertexAsKeyForLabelPropertyIndex(label, property, vertex.gid),
utils::SerializeVertexAsValueForLabelPropertyIndex(label, vertex.labels, vertex.properties),
CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::nullopt, ts), indexed_vertices->access());
utils::SerializeVertexAsValueForLabelPropertyIndex(label, vertex.labels, vertex.properties), vertex.delta,
indexed_vertices->access());
}
}
return gids;
@ -1354,7 +1349,7 @@ std::optional<storage::VertexAccessor> DiskStorage::LoadVertexToMainMemoryCache(
VertexAccessor DiskStorage::CreateVertexFromDisk(Transaction *transaction, utils::SkipList<Vertex>::Accessor &accessor,
storage::Gid gid, std::vector<LabelId> label_ids,
PropertyStore properties, Delta *delta) {
auto [it, inserted] = accessor.insert(Vertex{gid, delta});
auto [it, inserted] = accessor.insert(Vertex{gid, delta, OnDiskStorageTag{}});
MG_ASSERT(inserted, "The vertex must be inserted here!");
MG_ASSERT(it != accessor.end(), "Invalid Vertex accessor!");
it->labels = std::move(label_ids);

View File

@ -239,7 +239,7 @@ class DiskStorage final : public Storage {
Transaction *transaction, const std::string &key, const std::string &value, Delta *index_delta,
utils::SkipList<storage::Vertex>::Accessor index_accessor);
std::unordered_set<Gid> MergeVerticesFromMainCacheWithLabelIndexCache(Transaction *transaction, LabelId label,
View view, std::list<Delta> &index_deltas,
View view,
utils::SkipList<Vertex> *indexed_vertices);
/// Label-property-index
@ -247,7 +247,7 @@ class DiskStorage final : public Storage {
PropertyId property);
void HandleLoadingLabelPropertyForEdgeImportCache(Transaction *transaction, LabelId label, PropertyId property);
std::unordered_set<Gid> MergeVerticesFromMainCacheWithLabelPropertyIndexCache(
Transaction *transaction, LabelId label, PropertyId property, View view, std::list<Delta> &index_deltas,
Transaction *transaction, LabelId label, PropertyId property, View view,
utils::SkipList<Vertex> *indexed_vertices, const auto &label_property_filter);
void LoadVerticesFromDiskLabelPropertyIndex(Transaction *transaction, LabelId label, PropertyId property,
const std::unordered_set<storage::Gid> &gids,
@ -262,8 +262,7 @@ class DiskStorage final : public Storage {
std::unordered_set<Gid> MergeVerticesFromMainCacheWithLabelPropertyIndexCacheForIntervalSearch(
Transaction *transaction, LabelId label, PropertyId property, View view,
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
const std::optional<utils::Bound<PropertyValue>> &upper_bound, std::list<Delta> &index_deltas,
utils::SkipList<Vertex> *indexed_vertices);
const std::optional<utils::Bound<PropertyValue>> &upper_bound, utils::SkipList<Vertex> *indexed_vertices);
void LoadVerticesFromDiskLabelPropertyIndexForIntervalSearch(
Transaction *transaction, LabelId label, PropertyId property, const std::unordered_set<storage::Gid> &gids,
const std::optional<utils::Bound<PropertyValue>> &lower_bound,

View File

@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -23,7 +23,11 @@
namespace memgraph::storage {
struct OnDiskStorageTag {};
struct Vertex {
Vertex(Gid gid, Delta *delta, OnDiskStorageTag /*unused*/) : gid(gid), deleted(false), delta(delta) {}
Vertex(Gid gid, Delta *delta) : gid(gid), deleted(false), delta(delta) {
MG_ASSERT(delta == nullptr || delta->action == Delta::Action::DELETE_OBJECT ||
delta->action == Delta::Action::DELETE_DESERIALIZED_OBJECT,