From 826647c8761993b3a8166e6d3a868eafade5c6ff Mon Sep 17 00:00:00 2001 From: Andreja Tonev Date: Mon, 11 Mar 2024 17:58:17 +0100 Subject: [PATCH] BIG WIP: Using KVStore for ps; Used by edges --- src/query/cypher_query_interpreter.cpp | 2 +- src/query/interpreter.hpp | 1 + src/storage/v2/CMakeLists.txt | 1 + src/storage/v2/disk/storage.cpp | 3 +- src/storage/v2/durability/snapshot.cpp | 6 +- src/storage/v2/durability/wal.cpp | 4 +- src/storage/v2/edge.hpp | 83 ++++++- src/storage/v2/edge_accessor.cpp | 24 +- src/storage/v2/inmemory/storage.cpp | 2 +- src/storage/v2/property_disk_store.cpp | 32 +++ src/storage/v2/property_disk_store.hpp | 132 ++++++++++ src/storage/v2/storage.cpp | 3 + src/utils/rocksdb_serialization.hpp | 5 +- src/utils/skip_list.hpp | 30 ++- tests/unit/CMakeLists.txt | 3 + tests/unit/bfs_single_node.cpp | 10 +- tests/unit/pds.cpp | 328 +++++++++++++++++++++++++ 17 files changed, 630 insertions(+), 39 deletions(-) create mode 100644 src/storage/v2/property_disk_store.cpp create mode 100644 src/storage/v2/property_disk_store.hpp create mode 100644 tests/unit/pds.cpp diff --git a/src/query/cypher_query_interpreter.cpp b/src/query/cypher_query_interpreter.cpp index 30966119b..5d25cc551 100644 --- a/src/query/cypher_query_interpreter.cpp +++ b/src/query/cypher_query_interpreter.cpp @@ -68,7 +68,7 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map(stripped_query.query()); } catch (const SyntaxException &e) { // There is a syntax exception in the stripped query. Re-run the parser - // on the original query to get an appropriate error messsage. + // on the original query to get an appropriate error message. parser = std::make_unique(query_string); // If an exception was not thrown here, the stripper messed something diff --git a/src/query/interpreter.hpp b/src/query/interpreter.hpp index f18bd6721..b68be6adf 100644 --- a/src/query/interpreter.hpp +++ b/src/query/interpreter.hpp @@ -298,6 +298,7 @@ class Interpreter final { query_executions_.clear(); system_transaction_.reset(); transaction_queries_->clear(); + current_timeout_timer_.reset(); if (current_db_.db_acc_ && current_db_.db_acc_->is_deleting()) { current_db_.db_acc_.reset(); } diff --git a/src/storage/v2/CMakeLists.txt b/src/storage/v2/CMakeLists.txt index ec5108d63..f03d7d2a6 100644 --- a/src/storage/v2/CMakeLists.txt +++ b/src/storage/v2/CMakeLists.txt @@ -43,6 +43,7 @@ add_library(mg-storage-v2 STATIC replication/rpc.cpp replication/replication_storage_state.cpp inmemory/replication/recovery.cpp + property_disk_store.cpp ) target_link_libraries(mg-storage-v2 mg::replication Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk mg-events mg-memory) diff --git a/src/storage/v2/disk/storage.cpp b/src/storage/v2/disk/storage.cpp index 21fa5ecc7..4d9ea821f 100644 --- a/src/storage/v2/disk/storage.cpp +++ b/src/storage/v2/disk/storage.cpp @@ -1419,7 +1419,8 @@ std::optional DiskStorage::CreateEdgeFromDisk(const VertexAccessor MG_ASSERT(it != acc.end(), "Invalid Edge accessor!"); edge = EdgeRef(&*it); delta->prev.Set(&*it); - edge.ptr->properties.SetBuffer(properties); + // TODO Re-enable + // edge.ptr->properties.SetBuffer(properties); } ModifiedEdgeInfo modified_edge(Delta::Action::DELETE_DESERIALIZED_OBJECT, from_vertex->gid, to_vertex->gid, edge_type, diff --git a/src/storage/v2/durability/snapshot.cpp b/src/storage/v2/durability/snapshot.cpp index 5fea3dfa5..f48fdc927 100644 --- a/src/storage/v2/durability/snapshot.cpp +++ b/src/storage/v2/durability/snapshot.cpp @@ -277,7 +277,6 @@ void LoadPartialEdges(const std::filesystem::path &path, utils::SkipList & { auto props_size = snapshot.ReadUint(); if (!props_size) throw RecoveryFailure("Couldn't read the size of edge properties!"); - auto &props = it->properties; read_properties.clear(); read_properties.reserve(*props_size); for (uint64_t j = 0; j < *props_size; ++j) { @@ -287,7 +286,7 @@ void LoadPartialEdges(const std::filesystem::path &path, utils::SkipList & if (!value) throw RecoveryFailure("Couldn't read edge property value!"); read_properties.emplace_back(get_property_from_id(*key), std::move(*value)); } - props.InitProperties(std::move(read_properties)); + it->InitProperties(std::move(read_properties)); } } else { spdlog::debug("Ensuring edge {} doesn't have any properties.", *gid); @@ -720,7 +719,6 @@ RecoveredSnapshot LoadSnapshotVersion14(const std::filesystem::path &path, utils { auto props_size = snapshot.ReadUint(); if (!props_size) throw RecoveryFailure("Couldn't read the size of properties!"); - auto &props = it->properties; for (uint64_t j = 0; j < *props_size; ++j) { auto key = snapshot.ReadUint(); if (!key) throw RecoveryFailure("Couldn't read edge property id!"); @@ -728,7 +726,7 @@ RecoveredSnapshot LoadSnapshotVersion14(const std::filesystem::path &path, utils if (!value) throw RecoveryFailure("Couldn't read edge property value!"); SPDLOG_TRACE("Recovered property \"{}\" with value \"{}\" for edge {}.", name_id_mapper->IdToName(snapshot_id_map.at(*key)), *value, *gid); - props.SetProperty(get_property_from_id(*key), *value); + it->SetProperty(get_property_from_id(*key), *value); } } } else { diff --git a/src/storage/v2/durability/wal.cpp b/src/storage/v2/durability/wal.cpp index 5c40ab1c5..69ed6b10d 100644 --- a/src/storage/v2/durability/wal.cpp +++ b/src/storage/v2/durability/wal.cpp @@ -646,7 +646,7 @@ void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, const Delta // TODO (mferencevic): Mitigate the memory allocation introduced here // (with the `GetProperty` call). It is the only memory allocation in the // entire WAL file writing logic. - encoder->WritePropertyValue(edge.properties.GetProperty(delta.property.key)); + encoder->WritePropertyValue(edge.GetProperty(delta.property.key)); break; } case Delta::Action::DELETE_DESERIALIZED_OBJECT: @@ -926,7 +926,7 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst if (edge == edge_acc.end()) throw RecoveryFailure("The edge doesn't exist!"); auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.vertex_edge_set_property.property)); auto &property_value = delta.vertex_edge_set_property.value; - edge->properties.SetProperty(property_id, property_value); + edge->SetProperty(property_id, property_value); break; } case WalDeltaData::Type::TRANSACTION_END: diff --git a/src/storage/v2/edge.hpp b/src/storage/v2/edge.hpp index bdb224dfb..a1842e83b 100644 --- a/src/storage/v2/edge.hpp +++ b/src/storage/v2/edge.hpp @@ -1,4 +1,4 @@ -// Copyright 2023 Memgraph Ltd. +// Copyright 2024 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -19,6 +19,10 @@ #include "utils/logging.hpp" #include "utils/rw_spin_lock.hpp" +#include "storage/v2/property_disk_store.hpp" + +// #include "storage/v2/property_disk_store.hpp" + namespace memgraph::storage { struct Vertex; @@ -30,9 +34,14 @@ struct Edge { "Edge must be created with an initial DELETE_OBJECT delta!"); } + ~Edge() { + // TODO: Don't want to do this here + ClearProperties(); + } + Gid gid; - PropertyStore properties; + // PropertyStore properties; mutable utils::RWSpinLock lock; bool deleted; @@ -40,6 +49,76 @@ struct Edge { // uint16_t PAD; Delta *delta; + + // PSAPI Properties() { PDS::get(); } + + PropertyValue GetProperty(PropertyId property) const { + if (deleted) return {}; + const auto prop = PDS::get()->Get(gid, property); + if (prop) return *prop; + return {}; + } + + bool SetProperty(PropertyId property, const PropertyValue &value) { + if (deleted) return {}; + return PDS::get()->Set(gid, property, value); + } + + template + bool InitProperties(const TContainer &properties) { + if (deleted) return {}; + auto *pds = PDS::get(); + for (const auto &[property, value] : properties) { + if (value.IsNull()) { + continue; + } + if (!pds->Set(gid, property, value)) { + return false; + } + } + return true; + } + + void ClearProperties() { + auto *pds = PDS::get(); + pds->Clear(gid); + } + + std::map Properties() { + if (deleted) return {}; + return PDS::get()->Get(gid); + } + + std::vector> UpdateProperties( + std::map &properties) { + if (deleted) return {}; + auto old_properties = Properties(); + ClearProperties(); + + std::vector> id_old_new_change; + id_old_new_change.reserve(properties.size() + old_properties.size()); + for (const auto &[prop_id, new_value] : properties) { + if (!old_properties.contains(prop_id)) { + id_old_new_change.emplace_back(prop_id, PropertyValue(), new_value); + } + } + + for (const auto &[old_key, old_value] : old_properties) { + auto [it, inserted] = properties.emplace(old_key, old_value); + if (!inserted) { + auto &new_value = it->second; + id_old_new_change.emplace_back(it->first, old_value, new_value); + } + } + + MG_ASSERT(InitProperties(properties)); + return id_old_new_change; + } + + uint64_t PropertySize(PropertyId property) const { + if (deleted) return {}; + return PDS::get()->GetSize(gid, property); + } }; static_assert(alignof(Edge) >= 8, "The Edge should be aligned to at least 8!"); diff --git a/src/storage/v2/edge_accessor.cpp b/src/storage/v2/edge_accessor.cpp index 62a9f4bcd..76f77f4b5 100644 --- a/src/storage/v2/edge_accessor.cpp +++ b/src/storage/v2/edge_accessor.cpp @@ -128,11 +128,11 @@ Result EdgeAccessor::SetProperty(PropertyId property, co if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR; if (edge_.ptr->deleted) return Error::DELETED_OBJECT; - using ReturnType = decltype(edge_.ptr->properties.GetProperty(property)); + using ReturnType = decltype(edge_.ptr->GetProperty(property)); std::optional current_value; utils::AtomicMemoryBlock atomic_memory_block{ [¤t_value, &property, &value, transaction = transaction_, edge = edge_]() { - current_value.emplace(edge.ptr->properties.GetProperty(property)); + current_value.emplace(edge.ptr->GetProperty(property)); // We could skip setting the value if the previous one is the same to the new // one. This would save some memory as a delta would not be created as well as // avoid copying the value. The reason we are not doing that is because the @@ -140,7 +140,7 @@ Result EdgeAccessor::SetProperty(PropertyId property, co // "modify in-place". Additionally, the created delta will make other // transactions get a SERIALIZATION_ERROR. CreateAndLinkDelta(transaction, edge.ptr, Delta::SetPropertyTag(), property, *current_value); - edge.ptr->properties.SetProperty(property, value); + edge.ptr->SetProperty(property, value); }}; std::invoke(atomic_memory_block); @@ -162,7 +162,7 @@ Result EdgeAccessor::InitProperties(const std::mapdeleted) return Error::DELETED_OBJECT; - if (!edge_.ptr->properties.InitProperties(properties)) return false; + if (!edge_.ptr->InitProperties(properties)) return false; utils::AtomicMemoryBlock atomic_memory_block{[&properties, transaction_ = transaction_, edge_ = edge_]() { for (const auto &[property, _] : properties) { CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property, PropertyValue()); @@ -184,11 +184,11 @@ Result>> EdgeAc if (edge_.ptr->deleted) return Error::DELETED_OBJECT; - using ReturnType = decltype(edge_.ptr->properties.UpdateProperties(properties)); + using ReturnType = decltype(edge_.ptr->UpdateProperties(properties)); std::optional id_old_new_change; utils::AtomicMemoryBlock atomic_memory_block{ [transaction_ = transaction_, edge_ = edge_, &properties, &id_old_new_change]() { - id_old_new_change.emplace(edge_.ptr->properties.UpdateProperties(properties)); + id_old_new_change.emplace(edge_.ptr->UpdateProperties(properties)); for (auto &[property, old_value, new_value] : *id_old_new_change) { CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property, std::move(old_value)); } @@ -207,15 +207,15 @@ Result> EdgeAccessor::ClearProperties() { if (edge_.ptr->deleted) return Error::DELETED_OBJECT; - using ReturnType = decltype(edge_.ptr->properties.Properties()); + using ReturnType = decltype(edge_.ptr->Properties()); std::optional properties; utils::AtomicMemoryBlock atomic_memory_block{[&properties, transaction_ = transaction_, edge_ = edge_]() { - properties.emplace(edge_.ptr->properties.Properties()); + properties.emplace(edge_.ptr->Properties()); for (const auto &property : *properties) { CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property.first, property.second); } - edge_.ptr->properties.ClearProperties(); + edge_.ptr->ClearProperties(); }}; std::invoke(atomic_memory_block); @@ -231,7 +231,7 @@ Result EdgeAccessor::GetProperty(PropertyId property, View view) { auto guard = std::shared_lock{edge_.ptr->lock}; deleted = edge_.ptr->deleted; - value.emplace(edge_.ptr->properties.GetProperty(property)); + value.emplace(edge_.ptr->GetProperty(property)); delta = edge_.ptr->delta; } ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &value, property](const Delta &delta) { @@ -271,7 +271,7 @@ Result EdgeAccessor::GetPropertySize(PropertyId property, View view) c auto guard = std::shared_lock{edge_.ptr->lock}; Delta *delta = edge_.ptr->delta; if (!delta) { - return edge_.ptr->properties.PropertySize(property); + return edge_.ptr->PropertySize(property); } auto property_result = this->GetProperty(property, view); @@ -295,7 +295,7 @@ Result> EdgeAccessor::Properties(View view) { auto guard = std::shared_lock{edge_.ptr->lock}; deleted = edge_.ptr->deleted; - properties = edge_.ptr->properties.Properties(); + properties = edge_.ptr->Properties(); delta = edge_.ptr->delta; } ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &properties](const Delta &delta) { diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index dab56750b..bfcff9467 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -1146,7 +1146,7 @@ void InMemoryStorage::InMemoryAccessor::Abort() { current->timestamp->load(std::memory_order_acquire) == transaction_.transaction_id) { switch (current->action) { case Delta::Action::SET_PROPERTY: { - edge->properties.SetProperty(current->property.key, *current->property.value); + edge->SetProperty(current->property.key, *current->property.value); break; } case Delta::Action::DELETE_DESERIALIZED_OBJECT: diff --git a/src/storage/v2/property_disk_store.cpp b/src/storage/v2/property_disk_store.cpp new file mode 100644 index 000000000..cb85f19b4 --- /dev/null +++ b/src/storage/v2/property_disk_store.cpp @@ -0,0 +1,32 @@ +// Copyright 2024 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "storage/v2/property_disk_store.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/v2/temporal.hpp" +#include "utils/cast.hpp" +#include "utils/logging.hpp" + +namespace memgraph::storage { + +PDS *PDS::ptr_ = nullptr; + +} // namespace memgraph::storage diff --git a/src/storage/v2/property_disk_store.hpp b/src/storage/v2/property_disk_store.hpp new file mode 100644 index 000000000..1b8244105 --- /dev/null +++ b/src/storage/v2/property_disk_store.hpp @@ -0,0 +1,132 @@ +// Copyright 2024 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "kvstore/kvstore.hpp" +#include "slk/streams.hpp" +#include "storage/v2/id_types.hpp" +#include "storage/v2/property_value.hpp" + +#include "slk/serialization.hpp" +#include "storage/v2/replication/slk.hpp" + +namespace memgraph::storage { + +class PDS { + public: + static void Init(std::filesystem::path root) { + if (ptr_ == nullptr) ptr_ = new PDS(root); + } + + static PDS *get() { + if (ptr_ == nullptr) { + ptr_ = new PDS("/tmp"); + } + return ptr_; + } + + std::string ToKey(Gid gid, PropertyId pid) { + std::string key(sizeof(gid) + sizeof(pid), '\0'); + memcpy(key.data(), &gid, sizeof(gid)); + memcpy(&key[sizeof(gid)], &pid, sizeof(pid)); + return key; + } + + std::string ToPrefix(Gid gid) { + std::string key(sizeof(gid), '\0'); + memcpy(key.data(), &gid, sizeof(gid)); + return key; + } + + Gid ToGid(std::string_view sv) { + uint64_t gid; + gid = *((uint64_t *)sv.data()); + return Gid::FromUint(gid); + } + + PropertyId ToPid(std::string_view sv) { + uint32_t pid; + pid = *((uint32_t *)&sv[sizeof(Gid)]); + return PropertyId::FromUint(pid); + } + + PropertyValue ToPV(std::string_view sv) { + PropertyValue pv; + slk::Reader reader((const uint8_t *)sv.data(), sv.size()); + slk::Load(&pv, &reader); + return pv; + } + + std::string ToStr(const PropertyValue &pv) { + std::string val{}; + slk::Builder builder([&val](const uint8_t *data, size_t size, bool /*have_more*/) { + const auto old_size = val.size(); + val.resize(old_size + size); + memcpy(&val[old_size], data, size); + }); + slk::Save(pv, &builder); + builder.Finalize(); + return val; + } + + std::optional Get(Gid gid, PropertyId pid) { + const auto element = kvstore_.Get(ToKey(gid, pid)); + if (element) { + return ToPV(*element); + } + return std::nullopt; + } + + size_t GetSize(Gid gid, PropertyId pid) { + const auto element = kvstore_.Get(ToKey(gid, pid)); + if (element) { + return element->size(); + } + return 0; + } + + std::map Get(Gid gid) { + std::map res; + auto itr = kvstore_.begin(ToPrefix(gid)); + auto end = kvstore_.end(ToPrefix(gid)); + for (; itr != end; ++itr) { + if (!itr.IsValid()) continue; + res[ToPid(itr->first)] = ToPV(itr->second); + } + return res; + } + + auto Set(Gid gid, PropertyId pid, const PropertyValue &pv) { + if (pv.IsNull()) { + return kvstore_.Delete(ToKey(gid, pid)); + } + return kvstore_.Put(ToKey(gid, pid), ToStr(pv)); + } + + void Clear(Gid gid) { kvstore_.DeletePrefix(ToPrefix(gid)); } + + // kvstore::KVStore::iterator Itr() {} + + private: + PDS(std::filesystem::path root) : kvstore_{root / "pds"} {} + kvstore::KVStore kvstore_; + static PDS *ptr_; +}; + +} // namespace memgraph::storage diff --git a/src/storage/v2/storage.cpp b/src/storage/v2/storage.cpp index 536a504a0..5168b94b2 100644 --- a/src/storage/v2/storage.cpp +++ b/src/storage/v2/storage.cpp @@ -46,6 +46,9 @@ Storage::Storage(Config config, StorageMode storage_mode) indices_(config, storage_mode), constraints_(config, storage_mode) { spdlog::info("Created database with {} storage mode.", StorageModeToString(storage_mode)); + + // TODO: Make this work with MT + PDS::Init(config_.durability.storage_directory); } Storage::Accessor::Accessor(SharedAccess /* tag */, Storage *storage, IsolationLevel isolation_level, diff --git a/src/utils/rocksdb_serialization.hpp b/src/utils/rocksdb_serialization.hpp index 6871f1a69..1cf77faf4 100644 --- a/src/utils/rocksdb_serialization.hpp +++ b/src/utils/rocksdb_serialization.hpp @@ -1,4 +1,4 @@ -// Copyright 2023 Memgraph Ltd. +// Copyright 2024 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -143,7 +143,8 @@ inline std::string SerializeEdgeAsValue(const std::string &src_vertex_gid, const result += edge_type_str; result += "|"; if (edge) { - return result + utils::SerializeProperties(edge->properties); + // TODO: Re-enable + // return result + utils::SerializeProperties(edge->properties); } return result; } diff --git a/src/utils/skip_list.hpp b/src/utils/skip_list.hpp index 193d83b0b..70978e6e5 100644 --- a/src/utils/skip_list.hpp +++ b/src/utils/skip_list.hpp @@ -921,23 +921,27 @@ class SkipList final : detail::SkipListNode_base { } SkipList(SkipList &&other) noexcept : head_(other.head_), gc_(other.GetMemoryResource()), size_(other.size_.load()) { - other.head_ = nullptr; + if (this != &other) { + other.head_ = nullptr; + } } SkipList &operator=(SkipList &&other) noexcept { - MG_ASSERT(other.GetMemoryResource() == GetMemoryResource(), - "Move assignment with different MemoryResource is not supported"); - TNode *head = head_; - while (head != nullptr) { - TNode *succ = head->nexts[0].load(std::memory_order_acquire); - size_t bytes = SkipListNodeSize(*head); - head->~TNode(); - GetMemoryResource()->Deallocate(head, bytes); - head = succ; + if (this != &other) { + MG_ASSERT(other.GetMemoryResource() == GetMemoryResource(), + "Move assignment with different MemoryResource is not supported"); + TNode *head = head_; + while (head != nullptr) { + TNode *succ = head->nexts[0].load(std::memory_order_acquire); + size_t bytes = SkipListNodeSize(*head); + head->~TNode(); + GetMemoryResource()->Deallocate(head, bytes); + head = succ; + } + head_ = other.head_; + size_ = other.size_.load(); + other.head_ = nullptr; } - head_ = other.head_; - size_ = other.size_.load(); - other.head_ = nullptr; return *this; } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 44b24b6f6..e951025a5 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -452,3 +452,6 @@ add_unit_test(coordinator_cluster_state.cpp) target_link_libraries(${test_prefix}coordinator_cluster_state gflags mg-coordination mg-repl_coord_glue) target_include_directories(${test_prefix}coordinator_cluster_state PRIVATE ${CMAKE_SOURCE_DIR}/include) endif() + +add_unit_test(pds.cpp) +target_link_libraries(${test_prefix}pds mg-storage-v2) diff --git a/tests/unit/bfs_single_node.cpp b/tests/unit/bfs_single_node.cpp index a6816242d..a92b72901 100644 --- a/tests/unit/bfs_single_node.cpp +++ b/tests/unit/bfs_single_node.cpp @@ -9,9 +9,11 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +#include #include "bfs_common.hpp" #include "disk_test_utils.hpp" +#include "storage/v2/config.hpp" #include "storage/v2/disk/storage.hpp" #include "storage/v2/inmemory/storage.hpp" @@ -22,13 +24,19 @@ template class SingleNodeDb : public Database { public: const std::string testSuite = "bfs_single_node"; + const std::filesystem::path root_test = "/tmp/" + testSuite; - SingleNodeDb() : config_(disk_test_utils::GenerateOnDiskConfig(testSuite)), db_(new StorageType(config_)) {} + SingleNodeDb() : config_(disk_test_utils::GenerateOnDiskConfig(testSuite)), db_(new StorageType(config_)) { + memgraph::storage::UpdatePaths(config_, root_test); + } ~SingleNodeDb() override { if (std::is_same::value) { disk_test_utils::RemoveRocksDbDirs(testSuite); } + if (std::filesystem::exists(root_test)) { + std::filesystem::remove_all(root_test); + } } std::unique_ptr Access() override { diff --git a/tests/unit/pds.cpp b/tests/unit/pds.cpp new file mode 100644 index 000000000..346800aa4 --- /dev/null +++ b/tests/unit/pds.cpp @@ -0,0 +1,328 @@ +// Copyright 2024 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include + +#include + +#include "storage/v2/id_types.hpp" +#include "storage/v2/property_disk_store.hpp" +#include "storage/v2/property_value.hpp" +#include "storage/v2/temporal.hpp" + +const static std::filesystem::path test_root{"/tmp/MG_pds_test"}; + +class PdsTest : public ::testing::Test { + protected: + PdsTest() { memgraph::storage::PDS::Init(test_root); } + + ~PdsTest() override { + try { + if (std::filesystem::exists(test_root)) { + std::filesystem::remove_all(test_root); + } + } catch (...) { + } + } +}; + +TEST_F(PdsTest, Keys) { + using namespace memgraph::storage; + auto *pds = PDS::get(); + + auto gid = Gid::FromUint(13); + const auto gid_sv = pds->ToPrefix(gid); + EXPECT_TRUE(memcmp(&gid, gid_sv.data(), sizeof(uint64_t)) == 0); + EXPECT_EQ(gid, pds->ToGid(gid_sv)); + + auto pid = PropertyId::FromUint(243); + const auto key = pds->ToKey(gid, pid); + EXPECT_TRUE(memcmp(&gid, key.data(), sizeof(uint64_t)) == 0); + EXPECT_TRUE(memcmp(&pid, &key[sizeof(gid)], sizeof(uint32_t)) == 0); + EXPECT_EQ(pid, pds->ToPid(key)); +} + +TEST_F(PdsTest, BasicUsage) { + using namespace memgraph::storage; + auto *pds = PDS::get(); + + Gid gid; + PropertyId pid; + + PropertyValue pv_bf(false); + PropertyValue pv_bt(true); + PropertyValue pv_int0(0); + PropertyValue pv_int1(1); + PropertyValue pv_int16(16); + PropertyValue pv_640(int64_t(0)); + PropertyValue pv_641(int64_t(1)); + PropertyValue pv_64256(int64_t(256)); + PropertyValue pv_double0(0.0); + PropertyValue pv_double1(1.0); + PropertyValue pv_double1024(10.24); + PropertyValue pv_str(""); + PropertyValue pv_str0("0"); + PropertyValue pv_strabc("abc"); + PropertyValue pv_tdldt0(TemporalData(TemporalType::LocalDateTime, 0)); + PropertyValue pv_tdlt0(TemporalData(TemporalType::LocalTime, 0)); + PropertyValue pv_tdd0(TemporalData(TemporalType::Date, 0)); + PropertyValue pv_tddur0(TemporalData(TemporalType::Duration, 0)); + PropertyValue pv_tdldt1(TemporalData(TemporalType::LocalDateTime, 100000)); + PropertyValue pv_tdlt1(TemporalData(TemporalType::LocalTime, 100000)); + PropertyValue pv_tdd1(TemporalData(TemporalType::Date, 100000)); + PropertyValue pv_tddur1(TemporalData(TemporalType::Duration, 100000)); + PropertyValue pv_v(std::vector{PropertyValue(false), PropertyValue(1), PropertyValue(256), + PropertyValue(1.123), PropertyValue("")}); + PropertyValue pv_vv(std::vector{ + PropertyValue{std::vector{PropertyValue(false), PropertyValue(1), PropertyValue(256), + PropertyValue(1.123), PropertyValue("")}}, + PropertyValue{"string"}, PropertyValue{"list"}}); + + auto test = [&] { + { + ASSERT_TRUE(pds->Set(gid, pid, pv_bf)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsBool()); + ASSERT_FALSE(val->ValueBool()); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_bt)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsBool()); + ASSERT_TRUE(val->ValueBool()); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_int0)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsInt()); + ASSERT_EQ(val->ValueInt(), 0); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_int1)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsInt()); + ASSERT_EQ(val->ValueInt(), 1); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_int16)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsInt()); + ASSERT_EQ(val->ValueInt(), 16); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_640)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsInt()); + ASSERT_EQ(val->ValueInt(), 0); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_641)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsInt()); + ASSERT_EQ(val->ValueInt(), 1); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_64256)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsInt()); + ASSERT_EQ(val->ValueInt(), 256); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_double0)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsDouble()); + ASSERT_EQ(val->ValueDouble(), 0.0); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_double1)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsDouble()); + ASSERT_EQ(val->ValueDouble(), 1.0); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_double1024)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsDouble()); + ASSERT_EQ(val->ValueDouble(), 10.24); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_str)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsString()); + ASSERT_EQ(val->ValueString(), ""); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_str0)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsString()); + ASSERT_EQ(val->ValueString(), "0"); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_strabc)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsString()); + ASSERT_EQ(val->ValueString(), "abc"); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_tdd0)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsTemporalData()); + ASSERT_EQ(val->ValueTemporalData(), pv_tdd0.ValueTemporalData()); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_tdd1)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsTemporalData()); + ASSERT_EQ(val->ValueTemporalData(), pv_tdd1.ValueTemporalData()); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_tddur0)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsTemporalData()); + ASSERT_EQ(val->ValueTemporalData(), pv_tddur0.ValueTemporalData()); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_tddur1)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsTemporalData()); + ASSERT_EQ(val->ValueTemporalData(), pv_tddur1.ValueTemporalData()); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_tdldt0)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsTemporalData()); + ASSERT_EQ(val->ValueTemporalData(), pv_tdldt0.ValueTemporalData()); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_tdldt1)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsTemporalData()); + ASSERT_EQ(val->ValueTemporalData(), pv_tdldt1.ValueTemporalData()); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_tdlt0)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsTemporalData()); + ASSERT_EQ(val->ValueTemporalData(), pv_tdlt0.ValueTemporalData()); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_tdlt1)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsTemporalData()); + ASSERT_EQ(val->ValueTemporalData(), pv_tdlt1.ValueTemporalData()); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_v)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsList()); + const auto list = val->ValueList(); + ASSERT_EQ(list.size(), 5); + ASSERT_EQ(list[0], PropertyValue(false)); + ASSERT_EQ(list[1], PropertyValue(1)); + ASSERT_EQ(list[2], PropertyValue(256)); + ASSERT_EQ(list[3], PropertyValue(1.123)); + ASSERT_EQ(list[4], PropertyValue("")); + } + { + ASSERT_TRUE(pds->Set(gid, pid, pv_vv)); + const auto val = pds->Get(gid, pid); + ASSERT_TRUE(val); + ASSERT_TRUE(val->IsList()); + const auto list = val->ValueList(); + ASSERT_EQ(list.size(), 3); + { + const auto &val = list[0]; + ASSERT_TRUE(val.IsList()); + const auto list = val.ValueList(); + ASSERT_EQ(list.size(), 5); + ASSERT_EQ(list[0], PropertyValue(false)); + ASSERT_EQ(list[1], PropertyValue(1)); + ASSERT_EQ(list[2], PropertyValue(256)); + ASSERT_EQ(list[3], PropertyValue(1.123)); + ASSERT_EQ(list[4], PropertyValue("")); + } + ASSERT_EQ(list[1], PropertyValue("string")); + ASSERT_EQ(list[2], PropertyValue("list")); + } + }; + + gid.FromUint(0); + pid.FromUint(0); + test(); + + gid.FromUint(0); + pid.FromUint(1); + test(); + + gid.FromUint(1); + pid.FromUint(0); + test(); + + gid.FromUint(1); + pid.FromUint(1); + test(); + + gid.FromUint(0); + pid.FromUint(5446516); + test(); + + gid.FromUint(654645); + pid.FromUint(0); + test(); + + gid.FromUint(987615); + pid.FromUint(565); + test(); +} + +TEST_F(PdsTest, Get) { + using namespace memgraph::storage; + auto *pds = PDS::get(); + pds->Set(Gid::FromInt(0), PropertyId::FromInt(1), PropertyValue{"test1"}); + pds->Set(Gid::FromInt(0), PropertyId::FromInt(2), PropertyValue{"test2"}); + pds->Set(Gid::FromInt(0), PropertyId::FromInt(3), PropertyValue{"test3"}); + pds->Set(Gid::FromInt(1), PropertyId::FromInt(0), PropertyValue{"test0"}); + pds->Set(Gid::FromInt(1), PropertyId::FromInt(2), PropertyValue{"test02"}); + + auto all_0 = pds->Get(Gid::FromInt(0)); + ASSERT_EQ(all_0.size(), 3); + ASSERT_EQ(all_0[PropertyId::FromInt(1)], PropertyValue{"test1"}); + ASSERT_EQ(all_0[PropertyId::FromInt(2)], PropertyValue{"test2"}); + ASSERT_EQ(all_0[PropertyId::FromInt(3)], PropertyValue{"test3"}); +}