diff --git a/.gitignore b/.gitignore index 73bd3a458..0bde88a87 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ tags .gdb_history Testing/ ve/ +release/memgraph_* +release/libs/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 8bddd6850..9a1947eb7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -218,10 +218,12 @@ FILE(COPY ${include_dir}/storage/model/properties/string.hpp DESTINATION ${build FILE(COPY ${include_dir}/storage/model/properties/floating.hpp DESTINATION ${build_include_dir}/storage/model/properties) FILE(COPY ${include_dir}/storage/model/properties/number.hpp DESTINATION ${build_include_dir}/storage/model/properties) FILE(COPY ${include_dir}/storage/model/properties/integral.hpp DESTINATION ${build_include_dir}/storage/model/properties) +FILE(COPY ${include_dir}/storage/model/properties/property_family.hpp DESTINATION ${build_include_dir}/storage/model/properties) FILE(COPY ${include_dir}/storage/model/properties/utils/math_operations.hpp DESTINATION ${build_include_dir}/storage/model/properties/utils) FILE(COPY ${include_dir}/storage/model/properties/utils/unary_negation.hpp DESTINATION ${build_include_dir}/storage/model/properties/utils) FILE(COPY ${include_dir}/storage/model/properties/utils/modulo.hpp DESTINATION ${build_include_dir}/storage/model/properties/utils) + FILE(COPY ${include_dir}/storage/model/edge_model.hpp DESTINATION ${build_include_dir}/storage/model) FILE(COPY ${include_dir}/storage/model/property_model.hpp DESTINATION ${build_include_dir}/storage/model) FILE(COPY ${include_dir}/storage/model/vertex_model.hpp DESTINATION ${build_include_dir}/storage/model) @@ -407,6 +409,8 @@ set(memgraph_src_files ${src_dir}/storage/model/properties/bool.cpp ${src_dir}/storage/model/properties/string.cpp ${src_dir}/storage/model/properties/properties.cpp + ${src_dir}/storage/model/properties/property_family.cpp + ${src_dir}/storage/indexes/impl/nonunique_unordered_index.cpp ${src_dir}/storage/locking/record_lock.cpp ${src_dir}/storage/vertex_accessor.cpp ${src_dir}/transactions/transaction.cpp @@ -420,8 +424,8 @@ set(memgraph_src_files ${src_dir}/io/network/tls.cpp ${src_dir}/database/db.cpp ${src_dir}/database/db_accessor.cpp - ${src_dir}/database/db_transaction.cpp ${src_dir}/storage/edge_accessor.cpp + ${src_dir}/storage/record_accessor.cpp ) # STATIC library used by memgraph executables diff --git a/include/communication/bolt/v1/serialization/bolt_serializer.hpp b/include/communication/bolt/v1/serialization/bolt_serializer.hpp index f73f41d28..17f1e1206 100644 --- a/include/communication/bolt/v1/serialization/bolt_serializer.hpp +++ b/include/communication/bolt/v1/serialization/bolt_serializer.hpp @@ -81,9 +81,8 @@ public: // write the identifier for the node encoder.write_integer(edge.id()); - // TODO refactor when from() and to() start returning Accessors - encoder.write_integer(edge.from_record()->id); - encoder.write_integer(edge.to_record()->id); + encoder.write_integer(edge.from().id()); + encoder.write_integer(edge.to().id()); // write the type of the edge encoder.write_string(edge.edge_type()); diff --git a/include/data_structures/bitset/dynamic_bitset.hpp b/include/data_structures/bitset/dynamic_bitset.hpp index c228e9571..43024fade 100644 --- a/include/data_structures/bitset/dynamic_bitset.hpp +++ b/include/data_structures/bitset/dynamic_bitset.hpp @@ -1,7 +1,7 @@ #pragma once -#include #include +#include #include "threading/sync/lockable.hpp" #include "threading/sync/spinlock.hpp" @@ -13,8 +13,8 @@ class DynamicBitset : Lockable { Block() = default; - Block(Block&) = delete; - Block(Block&&) = delete; + Block(Block &) = delete; + Block(Block &&) = delete; static constexpr size_t size = sizeof(block_t) * 8; @@ -41,7 +41,7 @@ class DynamicBitset : Lockable block.fetch_and(~(bitmask(n) << k), order); } - std::atomic block {0}; + std::atomic block{0}; }; struct Chunk @@ -49,16 +49,13 @@ class DynamicBitset : Lockable Chunk() : next(nullptr) { static_assert(chunk_size % sizeof(block_t) == 0, - "chunk size not divisible by block size"); + "chunk size not divisible by block size"); } - Chunk(Chunk&) = delete; - Chunk(Chunk&&) = delete; + Chunk(Chunk &) = delete; + Chunk(Chunk &&) = delete; - ~Chunk() - { - delete next; - } + ~Chunk() { delete next; } static constexpr size_t size = chunk_size * Block::size; static constexpr size_t n_blocks = chunk_size / sizeof(block_t); @@ -79,54 +76,62 @@ class DynamicBitset : Lockable } Block blocks[n_blocks]; - std::atomic next; + std::atomic next; }; public: DynamicBitset() : head(new Chunk()) {} - DynamicBitset(DynamicBitset&) = delete; - DynamicBitset(DynamicBitset&&) = delete; + DynamicBitset(DynamicBitset &) = delete; + DynamicBitset(DynamicBitset &&) = delete; + + ~DynamicBitset() + { + auto now = head.load(); + while (now != nullptr) { + auto next = now->next.load(); + delete now; + now = next; + } + } block_t at(size_t k, size_t n) { - auto& chunk = find_chunk(k); + auto &chunk = find_chunk(k); return chunk.at(k, n, std::memory_order_seq_cst); } bool at(size_t k) { - auto& chunk = find_chunk(k); + auto &chunk = find_chunk(k); return chunk.at(k, 1, std::memory_order_seq_cst); } void set(size_t k, size_t n = 1) { - auto& chunk = find_chunk(k); + auto &chunk = find_chunk(k); return chunk.set(k, n, std::memory_order_seq_cst); } void clear(size_t k, size_t n = 1) { - auto& chunk = find_chunk(k); + auto &chunk = find_chunk(k); return chunk.clear(k, n, std::memory_order_seq_cst); } private: - Chunk& find_chunk(size_t& k) + Chunk &find_chunk(size_t &k) { - Chunk* chunk = head.load(), *next = nullptr; + Chunk *chunk = head.load(), *next = nullptr; // while i'm not in the right chunk // (my index is bigger than the size of this chunk) - while(k >= Chunk::size) - { + while (k >= Chunk::size) { next = chunk->next.load(); // if a next chunk exists, switch to it and decrement my // pointer by the size of the current chunk - if(next != nullptr) - { + if (next != nullptr) { chunk = next; k -= Chunk::size; continue; @@ -139,8 +144,7 @@ private: // double-check locking. if the chunk exists now, some other thread // has just created it, continue searching for my chunk - if(chunk->next.load() != nullptr) - continue; + if (chunk->next.load() != nullptr) continue; chunk->next.store(new Chunk()); } @@ -149,5 +153,5 @@ private: return *chunk; } - std::atomic head; + std::atomic head; }; diff --git a/include/data_structures/concurrent/concurrent_list.hpp b/include/data_structures/concurrent/concurrent_list.hpp new file mode 100644 index 000000000..76e547eda --- /dev/null +++ b/include/data_structures/concurrent/concurrent_list.hpp @@ -0,0 +1,283 @@ +#pragma once + +#include +#include +#include "utils/crtp.hpp" + +template +class List +{ +private: + template + static V load(std::atomic &atomic) + { + return atomic.load(std::memory_order_acquire); + } + + template + static void store(std::atomic &atomic, V desired) + { // Maybe could be relaxed + atomic.store(desired, std::memory_order_release); + } + + template + static bool cas(std::atomic &atomic, V expected, V desired) + { // Could be relaxed must be atleast Release. + return atomic.compare_exchange_strong(expected, desired, + std::memory_order_seq_cst); + } + + template + static V *swap(std::atomic &atomic, V *desired) + { // Could be relaxed + return atomic.exchange(desired, std::memory_order_seq_cst); + } + + class Node + { + public: + Node(const T &data) : data(data) {} + Node(T &&data) : data(std::move(data)) {} + + T data; + std::atomic next{nullptr}; + std::atomic next_rem{nullptr}; + std::atomic removed{false}; + }; + + template + class IteratorBase : public Crtp + { + friend class List; + + protected: + IteratorBase() : list(nullptr), curr(nullptr) {} + + IteratorBase(List *list) : list(list) + { + assert(list != nullptr); + list->count++; + reset(); + } + + public: + IteratorBase(const IteratorBase &) = delete; + + IteratorBase(IteratorBase &&other) + : list(other.list), curr(other.curr), prev(other.prev) + { + other.list = nullptr; + other.curr = nullptr; + } + + ~IteratorBase() + { + if (list == nullptr) { + return; + } + + auto head_rem = load(list->removed); + // Fetch could be relaxed + // There exist possibility that no one will delete garbage at this + // time. + if (list->count.fetch_sub(1) == 1 && head_rem != nullptr && + cas( + list->removed, head_rem, + nullptr)) { // I am the last one and there is garbage to be + // removed. + auto now = head_rem; + do { + auto next = load(now->next_rem); + delete now; + now = next; + } while (now != nullptr); + } + } + + T &operator*() const + { + assert(valid()); + return curr->data; + } + T *operator->() const + { + assert(valid()); + return &(curr->data); + } + + bool valid() const { return curr != nullptr; } + + // Iterating is wait free. + It &operator++() + { + assert(valid()); + do { + prev = curr; + curr = load(curr->next); + } while (valid() && is_removed()); + return this->derived(); + } + It &operator++(int) { return operator++(); } + + bool is_removed() + { + assert(valid()); + return load(curr->removed); + } + + // Returns IteratorBase to begining + void reset() + { + prev = nullptr; + curr = load(list->head); + while (valid() && is_removed()) { + operator++(); + } + } + + // Adds to the begining of list + // It is lock free but it isn't wait free. + void push(T &&data) + { + auto node = new Node(data); + Node *next = nullptr; + do { + next = load(list->head); + store(node->next, next); + } while (!cas(list->head, next, node)); + } + + // True only if this call removed the element. Only reason for fail is + // if + // the element is already removed. + // Remove has deadlock if another thread dies between marking node for + // removal + // and the disconnection. + // This can be improved with combinig the removed flag with prev.next or + // curr.next + bool remove() + { + assert(valid()); + if (cas(curr->removed, false, true)) { + if (!disconnect()) { + find_and_disconnect(); + } + store(curr->next_rem, swap(list->removed, curr)); + return true; + } + return false; + } + + friend bool operator==(const It &a, const It &b) + { + return a.curr == b.curr; + } + + friend bool operator!=(const It &a, const It &b) { return !(a == b); } + + private: + void find_and_disconnect() + { + auto it = It(list); + auto next = load(curr->next); + while (it.valid()) { + if (it.curr == curr) { + if (it.disconnect()) { + return; + } + it.reset(); + } else if (it.curr == next) { // Comparison with next is + // optimization for early return. + return; + } else { + it++; + } + } + } + + bool disconnect() + { + auto next = load(curr->next); + if (prev != nullptr) { + store(prev->next, next); + if (load(prev->removed)) { + return false; + } + } else if (!cas(list->head, curr, next)) { + return false; + } + return true; + } + + List *list; + Node *prev{nullptr}; + Node *curr; + }; + +public: + class ConstIterator : public IteratorBase + { + friend class List; + + public: + using IteratorBase::IteratorBase; + + const T &operator*() const + { + return IteratorBase::operator*(); + } + + const T *operator->() const + { + return IteratorBase::operator->(); + } + + operator const T &() const + { + return IteratorBase::operator T &(); + } + }; + + class Iterator : public IteratorBase + { + friend class List; + + public: + using IteratorBase::IteratorBase; + }; + +public: + List() = default; + + List(List &) = delete; + List(List &&) = delete; + + ~List() + { + auto now = head.load(); + while (now != nullptr) { + auto next = now->next.load(); + delete now; + now = next; + } + } + + void operator=(List &) = delete; + + Iterator begin() { return Iterator(this); } + + // ConstIterator begin() { return ConstIterator(this); } + + ConstIterator cbegin() { return ConstIterator(this); } + + Iterator end() { return Iterator(); } + + // ConstIterator end() { return ConstIterator(); } + + ConstIterator cend() { return ConstIterator(); } + +private: + std::atomic count{0}; + std::atomic head{nullptr}; + std::atomic removed{nullptr}; +}; diff --git a/include/data_structures/concurrent/concurrent_set.hpp b/include/data_structures/concurrent/concurrent_set.hpp index c550c081b..036463ad2 100644 --- a/include/data_structures/concurrent/concurrent_set.hpp +++ b/include/data_structures/concurrent/concurrent_set.hpp @@ -30,7 +30,7 @@ public: std::pair insert(T &&item) { - return accessor.insert(std::forward(item)); + return accessor.insert(std::move(item)); } list_it_con find(const T &item) const { return accessor.find(item); } diff --git a/include/data_structures/concurrent/skiplist.hpp b/include/data_structures/concurrent/skiplist.hpp index a0367a57c..2722cc343 100644 --- a/include/data_structures/concurrent/skiplist.hpp +++ b/include/data_structures/concurrent/skiplist.hpp @@ -157,7 +157,7 @@ public: // we have raw memory and we need to construct an object // of type Node on it - return new (node) Node(std::forward(item), height); + return new (node) Node(std::move(item), height); } static void destroy(Node *node) @@ -182,7 +182,7 @@ public: Node(T &&data, uint8_t height) : Node(height) { - this->data.set(std::forward(data)); + this->data.set(std::move(data)); } ~Node() @@ -522,7 +522,7 @@ public: std::pair insert(T &&item) { - return skiplist->insert(std::forward(item), preds, succs); + return skiplist->insert(std::move(item), preds, succs); } Iterator insert_non_unique(const T &item) @@ -683,7 +683,7 @@ private: static bool lock_nodes(uint8_t height, guard_t guards[], Node *preds[], Node *succs[]) { - Node *prepred, *pred, *succ = nullptr; + Node *prepred = nullptr, *pred = nullptr, *succ = nullptr; bool valid = true; for (int level = 0; valid && level < height; ++level) { @@ -790,8 +790,7 @@ private: // has the locks if (!lock_nodes(height, guards, preds, succs)) continue; - return {insert_here(std::forward(data), preds, succs, height, - guards), + return {insert_here(std::move(data), preds, succs, height, guards), true}; } } @@ -801,7 +800,7 @@ private: guard_t guards[]) { // you have the locks, create a new node - auto new_node = Node::create(std::forward(data), height); + auto new_node = Node::create(std::move(data), height); // link the predecessors and successors, e.g. // diff --git a/include/data_structures/map/rh_common.hpp b/include/data_structures/map/rh_common.hpp index 648666e56..244e1ec52 100644 --- a/include/data_structures/map/rh_common.hpp +++ b/include/data_structures/map/rh_common.hpp @@ -1,12 +1,13 @@ #pragma once -#include "utils/crtp.hpp" -#include "utils/option_ptr.hpp" +#include #include #include +#include "utils/crtp.hpp" +#include "utils/option_ptr.hpp" // RobinHood base. -// Entrys are POINTERS alligned to 8B. -// Entrys must know thers key. +// Entries are POINTERS alligned to 8B. +// Entries must know thers key. // D must have method K& get_key() // K must be comparable with ==. template @@ -186,22 +187,26 @@ public: RhBase() {} - RhBase(const RhBase &other) - { - capacity = other.capacity; - count = other.count; - if (capacity > 0) { - size_t bytes = sizeof(Combined) * capacity; - array = (Combined *)malloc(bytes); - memcpy(array, other.array, bytes); + RhBase(const RhBase &other) { copy_from(other); } - } else { - array = nullptr; - } - } + RhBase(RhBase &&other) { take_from(std::move(other)); } ~RhBase() { this->clear(); } + RhBase &operator=(const RhBase &other) + { + clear(); + copy_from(other); + return *this; + } + + RhBase &operator=(RhBase &&other) + { + clear(); + take_from(std::move(other)); + return *this; + } + Iterator begin() { return Iterator(this); } ConstIterator begin() const { return ConstIterator(this); } @@ -215,6 +220,30 @@ public: ConstIterator cend() const { return ConstIterator(); } protected: + void copy_from(const RhBase &other) + { + capacity = other.capacity; + count = other.count; + if (capacity > 0) { + size_t bytes = sizeof(Combined) * capacity; + array = (Combined *)malloc(bytes); + memcpy(array, other.array, bytes); + + } else { + array = nullptr; + } + } + + void take_from(RhBase &&other) + { + capacity = other.capacity; + count = other.count; + array = other.array; + other.array = nullptr; + other.count = 0; + other.capacity = 0; + } + void init_array(size_t size) { size_t bytes = sizeof(Combined) * size; @@ -240,6 +269,10 @@ protected: } Iterator create_it(size_t index) { return Iterator(this, index); } + ConstIterator create_it(size_t index) const + { + return ConstIterator(this, index); + } public: void clear() diff --git a/include/data_structures/map/rh_hashmultimap.hpp b/include/data_structures/map/rh_hashmultimap.hpp index 1c83244c1..23daa328b 100644 --- a/include/data_structures/map/rh_hashmultimap.hpp +++ b/include/data_structures/map/rh_hashmultimap.hpp @@ -2,11 +2,11 @@ #include #include - +#include "data_structures/map/rh_common.hpp" #include "utils/crtp.hpp" #include "utils/likely.hpp" +#include "utils/option.hpp" #include "utils/option_ptr.hpp" -#include "data_structures/map/rh_common.hpp" // HashMultiMap with RobinHood collision resolution policy. // Single threaded. @@ -48,9 +48,30 @@ public: using typename base::ConstIterator; using typename base::Iterator; - bool contains(const K &key) { return find(key) != end(); } + bool contains(const K &key) const { return find_index(key).is_present(); } Iterator find(const K &key_in) + { + auto index = find_index(key_in); + if (index) { + return create_it(index.get()); + } else { + return end(); + } + } + + ConstIterator find(const K &key_in) const + { + auto index = find_index(key_in); + if (index) { + return create_it(index.get()); + } else { + return end(); + } + } + +private: + Option find_index(const K &key_in) const { if (count > 0) { auto key = std::ref(key_in); @@ -62,7 +83,7 @@ public: while (other.valid() && off < border) { auto other_off = other.off(); if (other_off == off && key == other.ptr()->get_key()) { - return create_it(now); + return Option(now); } else if (other_off < off) { // Other is rich break; @@ -76,9 +97,10 @@ public: } } - return end(); + return Option(); } +public: // Inserts element. void add(D *data) { add(data->get_key(), data); } @@ -214,7 +236,7 @@ public: private: // Skips same key valus as other. true if whole map is full of same key // values. - bool skip(size_t &now, Combined &other, size_t other_off, size_t mask) + bool skip(size_t &now, Combined &other, size_t other_off, size_t mask) const { auto other_key = other.ptr()->get_key(); size_t start = now; diff --git a/include/database/db.hpp b/include/database/db.hpp index 626bb7dd7..028ed518e 100644 --- a/include/database/db.hpp +++ b/include/database/db.hpp @@ -1,7 +1,6 @@ #pragma once #include "storage/graph.hpp" -// #include "transactions/commit_log.hpp" #include "transactions/engine.hpp" class Db diff --git a/include/database/db_accessor.hpp b/include/database/db_accessor.hpp index 277a24171..7ffc27347 100644 --- a/include/database/db_accessor.hpp +++ b/include/database/db_accessor.hpp @@ -1,50 +1,98 @@ #pragma once -#include "database/db.hpp" -#include "database/db_accessor.hpp" -#include "storage/record_accessor.hpp" -#include "storage/vertex.hpp" +#include "database/db_transaction.hpp" #include "storage/vertex_accessor.hpp" -#include "storage/vertices.hpp" -#include "transactions/transaction.hpp" +#include "utils/border.hpp" +#include "utils/option.hpp" +namespace tx +{ +class Transaction; +} + +/* +* DbAccessor +* -Guarantees that access to Vertex and Edge is possible only through +* Vertex::Accessor and Edge::Accessor. +* -Guarantees that changing Vertex and Edge is possible only using +* Vertex::Accessor returned by vertex_insert() method and +* Edge::Accessor returned by edge_insert() method. +* -Offers CRUD for Vertex and Edge except iterating over all edges. +* +* Vertex::Accessor +* By default Vertex::accessor is empty. Caller has to call fill() method +* to fetch valid data and check it's return value. fill() method returns +* true if there is valid data for current transaction false otherwise. +* Only exception to this rule is vertex_insert() method in DbAccessor +* which returns by default filled Vertex::Accessor. +* +* Edge::Accessor +* By default Edge::accessor is empty. Caller has to call fill() method +* to +* fetch valid data and check it's return value. fill() method returns +* true +* if there is valid data for current transaction false otherwise. +* Only exception to this rule is edge_insert() method in DbAccessor +* which +* returns by default filled Edge::Accessor. +*/ class DbAccessor { + public: DbAccessor(Db &db); - // VERTEX METHODS - Vertices::vertices_t::Accessor vertex_access(); + //*******************VERTEX METHODS - const Vertex::Accessor vertex_find(const Id &id); + auto vertex_access(); - const Vertex::Accessor vertex_first(); + Option vertex_find(const Id &id); + // Creates new Vertex and returns filled Vertex::Accessor. Vertex::Accessor vertex_insert(); - // EDGE METHODS - Edge::Accessor edge_find(const Id &id); + // ******************* EDGE METHODS - Edge::Accessor edge_insert(VertexRecord *from, VertexRecord *to); + Option edge_find(const Id &id); + + // Creates new Edge and returns filled Edge::Accessor. + Edge::Accessor edge_insert(Vertex::Accessor const &from, + Vertex::Accessor const &to); + + // ******************* LABEL METHODS - // LABEL METHODS const Label &label_find_or_create(const std::string &name); bool label_contains(const std::string &name); - VertexIndexRecordCollection &label_find_index(const Label &label); + // ******************** TYPE METHODS - // TYPE METHODS const EdgeType &type_find_or_create(const std::string &name); bool type_contains(const std::string &name); - // TRANSACTION METHODS + // ******************** PROPERTY METHODS + + PropertyFamily &vertex_property_family_get(const std::string &name); + + PropertyFamily &edge_property_family_get(const std::string &name); + + // ******************** TRANSACTION METHODS + void commit(); void abort(); - // EASE OF USE METHODS - tx::Transaction &operator*(); +private: + template + friend class NonUniqueUnorderedIndex; - DbTransaction db; + DbTransaction db_transaction; }; + +// ********************** CONVENIENT FUNCTIONS + +template +bool option_fill(Option &o) +{ + return o.is_present() && o.get().fill(); +} diff --git a/include/database/db_transaction.hpp b/include/database/db_transaction.hpp index 5ddbbbc23..a36dd1d18 100644 --- a/include/database/db_transaction.hpp +++ b/include/database/db_transaction.hpp @@ -1,7 +1,5 @@ #pragma once -#include "storage/indexes/index_record.hpp" -#include "storage/label/label.hpp" #include "transactions/transaction.hpp" class Db; @@ -9,6 +7,8 @@ class DbAccessor; // Inner structures local to transaction can hold ref to this structure and use // its methods. +// Also serves as a barrier for calling methods defined public but meant for +// internal use. That kind of method should request DbTransaction&. class DbTransaction { friend DbAccessor; @@ -16,10 +16,11 @@ class DbTransaction public: DbTransaction(Db &db, tx::Transaction &trans) : db(db), trans(trans) {} - void update_label_index(const Label &label, - VertexIndexRecord &&index_record); - // protected: - // TRANSACTION METHODS + // Global transactional algorithms,operations and general methods meant for + // internal use should be here or should be routed through this object. + // This should provide cleaner hierarchy of operations on database. + // For example cleaner. + tx::Transaction &trans; Db &db; diff --git a/include/query_engine/hardcode/queries.hpp b/include/query_engine/hardcode/queries.hpp index 9bce411e2..51a0a4e5b 100644 --- a/include/query_engine/hardcode/queries.hpp +++ b/include/query_engine/hardcode/queries.hpp @@ -1,11 +1,15 @@ #pragma once #include "database/db.hpp" +#include "database/db_accessor.cpp" #include "database/db_accessor.hpp" #include "query_engine/query_stripper.hpp" #include "query_engine/util.hpp" +#include "storage/indexes/impl/nonunique_unordered_index.cpp" #include "storage/model/properties/property.hpp" +#include "storage/model/properties/property_family.hpp" #include "utils/command_line/arguments.hpp" +#include "utils/iterator/iterator.hpp" auto load_queries(Db &db) { @@ -14,8 +18,12 @@ auto load_queries(Db &db) // CREATE (n {prop: 0}) RETURN n) auto create_node = [&db](const properties_t &args) { DbAccessor t(db); + auto prop_key = t.vertex_property_family_get("prop") + .get(args[0]->flags) + .family_key(); + auto vertex_accessor = t.vertex_insert(); - vertex_accessor.property("prop", args[0]); + vertex_accessor.set(prop_key, args[0]); t.commit(); return true; }; @@ -23,8 +31,12 @@ auto load_queries(Db &db) auto create_labeled_and_named_node = [&db](const properties_t &args) { DbAccessor t(db); + auto prop_key = t.vertex_property_family_get("name") + .get(args[0]->flags) + .family_key(); + auto vertex_accessor = t.vertex_insert(); - vertex_accessor.property("name", args[0]); + vertex_accessor.set(prop_key, args[0]); auto &label = t.label_find_or_create("LABEL"); vertex_accessor.add_label(label); cout_properties(vertex_accessor.properties()); @@ -34,11 +46,23 @@ auto load_queries(Db &db) auto create_account = [&db](const properties_t &args) { DbAccessor t(db); + auto prop_id = + t.vertex_property_family_get("id").get(args[0]->flags).family_key(); + auto prop_name = t.vertex_property_family_get("name") + .get(args[1]->flags) + .family_key(); + auto prop_country = t.vertex_property_family_get("country") + .get(args[2]->flags) + .family_key(); + auto prop_created = t.vertex_property_family_get("created_at") + .get(args[3]->flags) + .family_key(); + auto vertex_accessor = t.vertex_insert(); - vertex_accessor.property("id", args[0]); - vertex_accessor.property("name", args[1]); - vertex_accessor.property("country", args[2]); - vertex_accessor.property("created_at", args[3]); + vertex_accessor.set(prop_id, args[0]); + vertex_accessor.set(prop_name, args[1]); + vertex_accessor.set(prop_country, args[2]); + vertex_accessor.set(prop_created, args[3]); auto &label = t.label_find_or_create("ACCOUNT"); vertex_accessor.add_label(label); cout_properties(vertex_accessor.properties()); @@ -48,13 +72,13 @@ auto load_queries(Db &db) auto find_node_by_internal_id = [&db](const properties_t &args) { DbAccessor t(db); - auto id = static_cast(*args[0]); - auto vertex_accessor = t.vertex_find(Id(id.value)); - if (!vertex_accessor) { + auto maybe_va = t.vertex_find(Id(args[0]->as().value)); + if (!option_fill(maybe_va)) { cout << "vertex doesn't exist" << endl; t.commit(); return false; } + auto vertex_accessor = maybe_va.get(); cout_properties(vertex_accessor.properties()); cout << "LABELS:" << endl; for (auto label_ref : vertex_accessor.labels()) { @@ -68,12 +92,12 @@ auto load_queries(Db &db) DbAccessor t(db); auto v1 = t.vertex_find(args[0]->as().value); - if (!v1) return t.commit(), false; + if (!option_fill(v1)) return t.commit(), false; auto v2 = t.vertex_find(args[1]->as().value); - if (!v2) return t.commit(), false; + if (!option_fill(v2)) return t.commit(), false; - auto edge_accessor = t.edge_insert(v1.vlist, v2.vlist); + auto edge_accessor = t.edge_insert(v1.get(), v2.get()); auto &edge_type = t.type_find_or_create("IS"); edge_accessor.edge_type(edge_type); @@ -89,19 +113,24 @@ auto load_queries(Db &db) auto find_edge_by_internal_id = [&db](const properties_t &args) { DbAccessor t(db); - auto edge_accessor = t.edge_find(args[0]->as().value); - if (!edge_accessor) return t.commit(), false; + auto maybe_ea = t.edge_find(args[0]->as().value); + if (!option_fill(maybe_ea)) return t.commit(), false; + auto edge_accessor = maybe_ea.get(); // print edge type and properties cout << "EDGE_TYPE: " << edge_accessor.edge_type() << endl; auto from = edge_accessor.from(); + if (!from.fill()) return t.commit(), false; + cout << "FROM:" << endl; - cout_properties(from.record->data.props); + cout_properties(from->data.props); auto to = edge_accessor.to(); + if (!to.fill()) return t.commit(), false; + cout << "TO:" << endl; - cout_properties(to.record->data.props); + cout_properties(to->data.props); t.commit(); @@ -110,11 +139,15 @@ auto load_queries(Db &db) auto update_node = [&db](const properties_t &args) { DbAccessor t(db); + auto prop_name = t.vertex_property_family_get("name") + .get(args[1]->flags) + .family_key(); - auto v = t.vertex_find(args[0]->as().value); - if (!v) return t.commit(), false; + auto maybe_v = t.vertex_find(args[0]->as().value); + if (!option_fill(maybe_v)) return t.commit(), false; + auto v = maybe_v.get(); - v.property("name", args[1]); + v.set(prop_name, args[1]); cout_properties(v.properties()); t.commit(); @@ -126,13 +159,20 @@ auto load_queries(Db &db) // weight: 70}]-(n2) RETURN r auto create_edge_v2 = [&db](const properties_t &args) { DbAccessor t(db); + + auto prop_age = + t.edge_property_family_get("age").get(args[2]->flags).family_key(); + auto prop_weight = t.edge_property_family_get("weight") + .get(args[3]->flags) + .family_key(); + auto n1 = t.vertex_find(args[0]->as().value); - if (!n1) return t.commit(), false; + if (!option_fill(n1)) return t.commit(), false; auto n2 = t.vertex_find(args[1]->as().value); - if (!n2) return t.commit(), false; - auto r = t.edge_insert(n2.vlist, n1.vlist); - r.property("age", args[2]); - r.property("weight", args[3]); + if (!option_fill(n2)) return t.commit(), false; + auto r = t.edge_insert(n2.get(), n1.get()); + r.set(prop_age, args[2]); + r.set(prop_weight, args[3]); auto &IS = t.type_find_or_create("IS"); r.edge_type(IS); @@ -145,15 +185,11 @@ auto load_queries(Db &db) auto match_all_nodes = [&db](const properties_t &args) { DbAccessor t(db); - auto vertices_accessor = t.vertex_access(); - for (auto &it : vertices_accessor) { - auto vertex = it.second.find(*t); - if (vertex == nullptr) continue; - cout_properties(vertex->data.props); - } - - // TODO - // db.graph.vertices.filter().all(t, handler); + iter::for_all(t.vertex_access(), [&](auto vertex) { + if (vertex.fill()) { + cout_properties(vertex->data.props); + } + }); t.commit(); @@ -166,21 +202,17 @@ auto load_queries(Db &db) DbAccessor t(db); auto &label = t.label_find_or_create("LABEL"); + auto prop_key = + t.vertex_property_family_get("name").get(Type::String).family_key(); - auto &index_record_collection = t.label_find_index(label); - auto accessor = index_record_collection.access(); cout << "VERTICES" << endl; - for (auto &v : accessor) { - cout << v.record->data.props.at("name").as().value << endl; - } - - // TODO - // db.graph.vertices.fileter("LABEL").all(t, handler); + iter::for_all(label.index->for_range_exact(t), + [&](auto a) { cout << a.at(prop_key) << endl; }); return true; }; - queries[4857652843629217005u] = find_by_label; + queries[4857652843629217005u] = find_by_label; queries[10597108978382323595u] = create_account; queries[5397556489557792025u] = create_labeled_and_named_node; queries[7939106225150551899u] = create_edge; diff --git a/include/query_engine/util.hpp b/include/query_engine/util.hpp index 76bf1e5a6..40b5b5a57 100644 --- a/include/query_engine/util.hpp +++ b/include/query_engine/util.hpp @@ -40,5 +40,4 @@ std::string code_line(const std::string &format_str, const Args &... args) { return "\t" + format(format_str, args...) + "\n"; } - } diff --git a/include/storage/edge_accessor.hpp b/include/storage/edge_accessor.hpp index b43612111..f62328eae 100644 --- a/include/storage/edge_accessor.hpp +++ b/include/storage/edge_accessor.hpp @@ -9,7 +9,6 @@ class Edges; -// TODO: Edge, Db, Edge::Accessor class Edge::Accessor : public RecordAccessor { public: @@ -22,8 +21,4 @@ public: Vertex::Accessor from() const; Vertex::Accessor to() const; - - VertexRecord *from_record() const; - - VertexRecord *to_record() const; }; diff --git a/include/storage/edge_record.hpp b/include/storage/edge_record.hpp index e04cde31b..aa1471abe 100644 --- a/include/storage/edge_record.hpp +++ b/include/storage/edge_record.hpp @@ -27,7 +27,7 @@ public: auto to() const { return this->to_v; } -private: +protected: VertexRecord *from_v; VertexRecord *to_v; }; diff --git a/include/storage/edges.hpp b/include/storage/edges.hpp index 5cc00b670..080d39836 100644 --- a/include/storage/edges.hpp +++ b/include/storage/edges.hpp @@ -1,18 +1,31 @@ #pragma once +#include #include "data_structures/concurrent/concurrent_map.hpp" #include "mvcc/version_list.hpp" #include "storage/common.hpp" #include "storage/edge_accessor.hpp" +#include "storage/model/properties/property_family.hpp" +#include "utils/option.hpp" class Edges { + using prop_familys_t = ConcurrentMap; + public: - Edge::Accessor find(DbTransaction &t, const Id &id); + Option find(DbTransaction &t, const Id &id); + + // Creates new Edge and returns filled Edge::Accessor. Edge::Accessor insert(DbTransaction &t, VertexRecord *from, VertexRecord *to); + PropertyFamily &property_family_find_or_create(const std::string &name); + private: ConcurrentMap edges; + // TODO: Because familys wont be removed this could be done with more + // efficent + // data structure. + prop_familys_t prop_familys; AtomicCounter counter; }; diff --git a/include/storage/indexes/impl/nonunique_unordered_index.hpp b/include/storage/indexes/impl/nonunique_unordered_index.hpp new file mode 100644 index 000000000..01bebb1c5 --- /dev/null +++ b/include/storage/indexes/impl/nonunique_unordered_index.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include "storage/indexes/index_base.hpp" +#include "storage/indexes/index_record.hpp" + +#include "data_structures/concurrent/concurrent_list.hpp" + +template +class NonUniqueUnorderedIndex : public IndexBase +{ +public: + typedef T value_type; + typedef K key_type; + + NonUniqueUnorderedIndex(); + + // Insert's value. + // nonunique => always succeds. + bool insert(IndexRecord &&value) final; + + // Returns iterator which returns valid records in range. + // ordered==None => doesn't guarantee any order of submitting records. + std::unique_ptr> + for_range(DbAccessor &t, Border from = Border(), + Border to = Border()) final; + + // Same as for_range just whit known returned iterator. + auto for_range_exact(DbAccessor &t, Border from = Border(), + Border to = Border()); + + // Removes for all transactions obsolete Records. + // Cleaner has to call this method when he decideds that it is time for + // cleaning. + void clean(DbTransaction &) final; + +private: + List> list; +}; diff --git a/include/storage/indexes/index.hpp b/include/storage/indexes/index.hpp index 4b24e65a3..87007f0a7 100644 --- a/include/storage/indexes/index.hpp +++ b/include/storage/indexes/index.hpp @@ -1,44 +1,44 @@ -#pragma once - -#include - -#include "data_structures/concurrent/concurrent_map.hpp" -#include "storage/indexes/index_record.hpp" -#include "storage/indexes/index_record_collection.hpp" -#include "storage/label/label.hpp" - -template -class Index -{ - public: - using container_t = ConcurrentMap; - - Index() : index(std::make_unique()) {} - - auto update(const Label &label, VertexIndexRecord &&index_record) - { - auto accessor = index->access(); - auto label_ref = label_ref_t(label); - - // create Index Record Collection if it doesn't exist - if (!accessor.contains(label_ref)) { - accessor.insert(label_ref, std::move(VertexIndexRecordCollection())); - } - - // add Vertex Index Record to the Record Collection - auto &record_collection = (*accessor.find(label_ref)).second; - record_collection.add(std::forward(index_record)); - } - - VertexIndexRecordCollection &find(const Label &label) - { - // TODO: accessor should be outside? - // bacause otherwise GC could delete record that has just be returned - auto label_ref = label_ref_t(label); - auto accessor = index->access(); - return (*accessor.find(label_ref)).second; - } - - private: - std::unique_ptr index; -}; +// #pragma once +// +// #include +// +// #include "data_structures/concurrent/concurrent_map.hpp" +// #include "storage/indexes/index_record.hpp" +// #include "storage/indexes/index_record_collection.hpp" +// #include "storage/label/label.hpp" +// +// template +// class Index +// { +// public: +// using container_t = ConcurrentMap; +// +// Index() : index(std::make_unique()) {} +// +// auto update(const Label &label, VertexIndexRecord &&index_record) +// { +// auto accessor = index->access(); +// auto label_ref = label_ref_t(label); +// +// // create Index Record Collection if it doesn't exist +// if (!accessor.contains(label_ref)) { +// accessor.insert(label_ref, std::move(VertexIndexRecordCollection())); +// } +// +// // add Vertex Index Record to the Record Collection +// auto &record_collection = (*accessor.find(label_ref)).second; +// record_collection.add(std::forward(index_record)); +// } +// +// VertexIndexRecordCollection &find(const Label &label) +// { +// // TODO: accessor should be outside? +// // bacause otherwise GC could delete record that has just be returned +// auto label_ref = label_ref_t(label); +// auto accessor = index->access(); +// return (*accessor.find(label_ref)).second; +// } +// +// private: +// std::unique_ptr index; +// }; diff --git a/include/storage/indexes/index_base.hpp b/include/storage/indexes/index_base.hpp new file mode 100644 index 000000000..aa7603848 --- /dev/null +++ b/include/storage/indexes/index_base.hpp @@ -0,0 +1,60 @@ +#pragma once + +// #include "storage/indexes/index_record.hpp" +#include +#include +#include "utils/border.hpp" +#include "utils/iterator/iterator_base.hpp" + +class DbTransaction; +class DbAccessor; + +template +class IndexRecord; + +// Defines ordering of data +enum Order +{ + None = 0, + Ascending = 1, + Descending = 2, +}; + +// Interface for all indexes. +// T type of record. +// K type of key on which records are ordered +template +class IndexBase +{ +public: + typedef T value_type; + typedef K key_type; + + IndexBase(bool unique, Order order) : unique(unique), order(order) {} + + // Insert's value. + // unique => returns false if there is already valid equal value. + // nonunique => always succeds. + virtual bool insert(IndexRecord &&value) = 0; + + // Returns iterator which returns valid records in range. + // order==noe => doesn't guarantee any order of returned records. + // order==Ascending => guarantees order of returnd records will be from + // smallest to largest. + // order==Descending => guarantees order of returned records will be from + // largest to smallest. + // Range must be from<=to + virtual std::unique_ptr> + for_range(DbAccessor &, Border from = Border(), + Border to = Border()) = 0; + + // Removes for all transactions obsolete Records. + // Cleaner has to call this method when he decideds that it is time for + // cleaning. + virtual void clean(DbTransaction &) = 0; + + // Are the records unique + const bool unique; + // Ordering of the records. + const Order order; +}; diff --git a/include/storage/indexes/index_record.hpp b/include/storage/indexes/index_record.hpp index b7e63efaa..70f05af71 100644 --- a/include/storage/indexes/index_record.hpp +++ b/include/storage/indexes/index_record.hpp @@ -1,46 +1,67 @@ #pragma once +#include "database/db_transaction.hpp" #include "mvcc/version_list.hpp" #include "utils/total_ordering.hpp" -template -class IndexRecord : TotalOrdering> +// class DbTransaction; +// namespace tx +// { +// class Transaction; +// } + +// T type of record. +// K key on which record is ordered. +template +class IndexRecord : public TotalOrdering> { public: using vlist_t = mvcc::VersionList; IndexRecord() = default; - IndexRecord(T *record, vlist_t *vlist) : record(record), vlist(vlist) + IndexRecord(K key, T *record, vlist_t *vlist) + : key(std::move(key)), record(record), vlist(vlist) { assert(record != nullptr); assert(vlist != nullptr); } - friend bool operator<(const IndexRecord& lhs, const IndexRecord& rhs) + friend bool operator<(const IndexRecord &lhs, const IndexRecord &rhs) { - return lhs.record < rhs.record; + return lhs.key < rhs.key || + (lhs.key == rhs.key && lhs.vlist == rhs.vlist && + lhs.record < rhs.record); } - friend bool operator==(const IndexRecord& lhs, const IndexRecord& rhs) + friend bool operator==(const IndexRecord &lhs, const IndexRecord &rhs) { - return lhs.record == rhs.record; + return lhs.key == rhs.key && + (lhs.vlist != rhs.vlist || lhs.record == rhs.record); } bool empty() const { return record == nullptr; } - // const typename T::Accessor get() - // { - // // TODO: if somebody wants to read T content - // // const T::Accessor has to be returned from here - // // the problem is that here we don't have pointer to store - // // TODO: figure it out - // } + bool is_valid(tx::Transaction &t) const + { + assert(!empty()); + return record == vlist->find(t); + } -// private: + const auto access(DbTransaction &db) const + { + return T::Accessor::create(record, vlist, db); + } + + const K key; + +private: T *const record{nullptr}; vlist_t *const vlist{nullptr}; }; -using VertexIndexRecord = IndexRecord; -using EdgeIndexRecord = IndexRecord; +template +using VertexIndexRecord = IndexRecord; + +template +using EdgeIndexRecord = IndexRecord; diff --git a/include/storage/indexes/index_record_collection.hpp b/include/storage/indexes/index_record_collection.hpp index 0d06e2a91..53d4a1ba3 100644 --- a/include/storage/indexes/index_record_collection.hpp +++ b/include/storage/indexes/index_record_collection.hpp @@ -1,38 +1,38 @@ -#pragma once - -#include - -#include "data_structures/concurrent/concurrent_set.hpp" -#include "storage/indexes/index_record.hpp" - -template -class IndexRecordCollection -{ -public: - using index_record_t = IndexRecord; - using index_record_collection_t = ConcurrentSet; - - IndexRecordCollection() - : records(std::make_unique()) - { - } - - void add(index_record_t &&record) - { - auto accessor = records->access(); - accessor.insert(std::forward(record)); - } - - auto access() - { - return records->access(); - } - - // TODO: iterator and proxy - -private: - std::unique_ptr records; -}; - -using VertexIndexRecordCollection = IndexRecordCollection; -using EdgeIndexRecordCollection = IndexRecordCollection; +// #pragma once +// +// #include +// +// #include "data_structures/concurrent/concurrent_set.hpp" +// #include "storage/indexes/index_record.hpp" +// +// template +// class IndexRecordCollection +// { +// public: +// using index_record_t = IndexRecord; +// using index_record_collection_t = ConcurrentSet; +// +// IndexRecordCollection() +// : records(std::make_unique()) +// { +// } +// +// void add(index_record_t &&record) +// { +// auto accessor = records->access(); +// accessor.insert(std::forward(record)); +// } +// +// auto access() +// { +// return records->access(); +// } +// +// // TODO: iterator and proxy +// +// private: +// std::unique_ptr records; +// }; +// +// using VertexIndexRecordCollection = IndexRecordCollection; +// using EdgeIndexRecordCollection = IndexRecordCollection; diff --git a/include/storage/indexes/sort_order.hpp b/include/storage/indexes/sort_order.hpp index 82a60f61d..5d9348601 100644 --- a/include/storage/indexes/sort_order.hpp +++ b/include/storage/indexes/sort_order.hpp @@ -3,7 +3,7 @@ template struct Ascending { - constexpr bool operator()(const T& lhs, const T& rhs) const + constexpr bool operator()(const T &lhs, const T &rhs) const { return lhs < rhs; } @@ -12,7 +12,7 @@ struct Ascending template struct Descending { - constexpr bool operator()(const T& lhs, const T& rhs) const + constexpr bool operator()(const T &lhs, const T &rhs) const { return lhs > rhs; } diff --git a/include/storage/label/label.hpp b/include/storage/label/label.hpp index 97430fe5a..bcb478921 100644 --- a/include/storage/label/label.hpp +++ b/include/storage/label/label.hpp @@ -1,27 +1,39 @@ #pragma once -#include #include +#include -#include "utils/total_ordering.hpp" +#include "storage/indexes/impl/nonunique_unordered_index.hpp" +#include "storage/vertex.hpp" +#include "storage/vertex_accessor.hpp" #include "utils/reference_wrapper.hpp" +#include "utils/total_ordering.hpp" +#include "utils/void.hpp" + +using LabelIndexRecord = VertexIndexRecord; class Label : public TotalOrdering