diff --git a/CMakeLists.txt b/CMakeLists.txt index 40dc465c8..0813af66a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -429,8 +429,14 @@ set(memgraph_src_files ${src_dir}/storage/model/properties/array.cpp ${src_dir}/storage/model/properties/properties.cpp ${src_dir}/storage/model/properties/property_family.cpp + ${src_dir}/storage/indexes/index_base.cpp + ${src_dir}/storage/indexes/index_record.cpp + ${src_dir}/storage/indexes/index_update.cpp + ${src_dir}/storage/indexes/index_holder.cpp + ${src_dir}/storage/indexes/impl/unique_ordered_index.cpp ${src_dir}/storage/indexes/impl/nonunique_unordered_index.cpp ${src_dir}/storage/locking/record_lock.cpp + ${src_dir}/storage/garbage/garbage.cpp ${src_dir}/storage/vertex_accessor.cpp ${src_dir}/transactions/transaction.cpp ${src_dir}/template_engine/engine.cpp @@ -442,6 +448,7 @@ set(memgraph_src_files ${src_dir}/logging/log.cpp ${src_dir}/io/network/tls.cpp ${src_dir}/database/db.cpp + ${src_dir}/database/db_transaction.cpp ${src_dir}/database/db_accessor.cpp ${src_dir}/storage/edge_accessor.cpp ${src_dir}/storage/record_accessor.cpp diff --git a/include/communication/bolt/v1/serialization/bolt_serializer.hpp b/include/communication/bolt/v1/serialization/bolt_serializer.hpp index d05609645..4e01b7948 100644 --- a/include/communication/bolt/v1/serialization/bolt_serializer.hpp +++ b/include/communication/bolt/v1/serialization/bolt_serializer.hpp @@ -33,7 +33,7 @@ public: * } * */ - void write(const Vertex::Accessor &vertex) + void write(const VertexAccessor &vertex) { // write signatures for the node struct and node data type encoder.write_struct_header(3); @@ -72,7 +72,7 @@ public: * } * */ - void write(const Edge::Accessor &edge) + void write(const EdgeAccessor &edge) { // write signatures for the edge struct and edge data type encoder.write_struct_header(5); diff --git a/include/communication/bolt/v1/serialization/record_stream.hpp b/include/communication/bolt/v1/serialization/record_stream.hpp index aa4ab438f..39c5534ea 100644 --- a/include/communication/bolt/v1/serialization/record_stream.hpp +++ b/include/communication/bolt/v1/serialization/record_stream.hpp @@ -78,8 +78,8 @@ public: } // -- BOLT SPECIFIC METHODS ----------------------------------------------- - void write(const Vertex::Accessor &vertex) { serializer.write(vertex); } - void write(const Edge::Accessor &edge) { serializer.write(edge); } + void write(const VertexAccessor &vertex) { serializer.write(vertex); } + void write(const EdgeAccessor &edge) { serializer.write(edge); } void write(const Property &prop) { serializer.write(prop); } void write(const Bool& prop) { serializer.write(prop); } diff --git a/include/data_structures/concurrent/concurrent_list.hpp b/include/data_structures/concurrent/concurrent_list.hpp index 76e547eda..04e221220 100644 --- a/include/data_structures/concurrent/concurrent_list.hpp +++ b/include/data_structures/concurrent/concurrent_list.hpp @@ -2,6 +2,7 @@ #include <atomic> #include <cassert> +#include <utility> #include "utils/crtp.hpp" template <class T> diff --git a/include/data_structures/concurrent/concurrent_map.hpp b/include/data_structures/concurrent/concurrent_map.hpp index 4e751a109..ae24568f1 100644 --- a/include/data_structures/concurrent/concurrent_map.hpp +++ b/include/data_structures/concurrent/concurrent_map.hpp @@ -8,53 +8,66 @@ using std::pair; template <typename K, typename T> class ConcurrentMap { - typedef Item<K, T> item_t; - typedef SkipList<item_t> list; - typedef typename SkipList<item_t>::Iterator list_it; - typedef typename SkipList<item_t>::ConstIterator list_it_con; - - public: - ConcurrentMap() {} - - class Accessor : public AccessorBase<item_t> - { - friend class ConcurrentMap; - - using AccessorBase<item_t>::AccessorBase; - -private: - using AccessorBase<item_t>::accessor; + typedef Item<K, T> item_t; + typedef SkipList<item_t> list; + typedef typename SkipList<item_t>::Iterator list_it; + typedef typename SkipList<item_t>::ConstIterator list_it_con; public: - std::pair<list_it, bool> insert(const K &key, const T &data) + ConcurrentMap() {} + + class Accessor : public AccessorBase<item_t> { - return accessor.insert(item_t(key, data)); - } + friend class ConcurrentMap; - std::pair<list_it, bool> insert(const K &key, T &&data) - { - return accessor.insert(item_t(key, std::forward<T>(data))); - } + using AccessorBase<item_t>::AccessorBase; - std::pair<list_it, bool> insert(K &&key, T &&data) - { - return accessor.insert( - item_t(std::forward<K>(key), std::forward<T>(data))); - } + private: + using AccessorBase<item_t>::accessor; - list_it_con find(const K &key) const { return accessor.find(key); } + public: + std::pair<list_it, bool> insert(const K &key, const T &data) + { + return accessor.insert(item_t(key, data)); + } - list_it find(const K &key) { return accessor.find(key); } + std::pair<list_it, bool> insert(const K &key, T &&data) + { + return accessor.insert(item_t(key, std::forward<T>(data))); + } - bool contains(const K &key) const { return this->find(key) != this->end(); } + std::pair<list_it, bool> insert(K &&key, T &&data) + { + return accessor.insert( + item_t(std::forward<K>(key), std::forward<T>(data))); + } - bool remove(const K &key) { return accessor.remove(key); } - }; + list_it_con find(const K &key) const { return accessor.find(key); } - Accessor access() { return Accessor(&skiplist); } + list_it find(const K &key) { return accessor.find(key); } - const Accessor access() const { return Accessor(&skiplist); } + list_it_con find_or_larger(const T &item) const + { + return accessor.find_or_larger(item); + } - private: - list skiplist; + list_it find_or_larger(const T &item) + { + return accessor.find_or_larger(item); + } + + bool contains(const K &key) const + { + return this->find(key) != this->end(); + } + + bool remove(const K &key) { return accessor.remove(key); } + }; + + Accessor access() { return Accessor(&skiplist); } + + const Accessor access() const { return Accessor(&skiplist); } + +private: + list skiplist; }; diff --git a/include/data_structures/concurrent/concurrent_multimap.hpp b/include/data_structures/concurrent/concurrent_multimap.hpp index cddce9914..1ae53ce75 100644 --- a/include/data_structures/concurrent/concurrent_multimap.hpp +++ b/include/data_structures/concurrent/concurrent_multimap.hpp @@ -53,6 +53,16 @@ public: list_it find(const K &key) { return accessor.find(key); } + list_it_con find_or_larger(const T &item) const + { + return accessor.find_or_larger(item); + } + + list_it find_or_larger(const T &item) + { + return accessor.find_or_larger(item); + } + bool contains(const K &key) const { return this->find(key) != this->end(); diff --git a/include/data_structures/concurrent/concurrent_multiset.hpp b/include/data_structures/concurrent/concurrent_multiset.hpp index eae01c2d0..858620b1c 100644 --- a/include/data_structures/concurrent/concurrent_multiset.hpp +++ b/include/data_structures/concurrent/concurrent_multiset.hpp @@ -5,46 +5,59 @@ template <class T> class ConcurrentMultiSet { - typedef SkipList<T> list; - typedef typename SkipList<T>::Iterator list_it; - typedef typename SkipList<T>::ConstIterator list_it_con; - - public: - ConcurrentMultiSet() {} - - class Accessor : public AccessorBase<T> - { - friend class ConcurrentMultiSet; - - using AccessorBase<T>::AccessorBase; - -private: - using AccessorBase<T>::accessor; + typedef SkipList<T> list; + typedef typename SkipList<T>::Iterator list_it; + typedef typename SkipList<T>::ConstIterator list_it_con; public: - list_it insert(const T &item) { return accessor.insert_non_unique(item); } + ConcurrentMultiSet() {} - list_it insert(T &&item) + class Accessor : public AccessorBase<T> { - return accessor.insert_non_unique(std::forward<T>(item)); - } + friend class ConcurrentMultiSet; - list_it_con find(const T &item) const { return accessor.find(item); } + using AccessorBase<T>::AccessorBase; - list_it find(const T &item) { return accessor.find(item); } + private: + using AccessorBase<T>::accessor; - bool contains(const T &item) const - { - return this->find(item) != this->end(); - } + public: + list_it insert(const T &item) + { + return accessor.insert_non_unique(item); + } - bool remove(const T &item) { return accessor.remove(item); } - }; + list_it insert(T &&item) + { + return accessor.insert_non_unique(std::forward<T>(item)); + } - Accessor access() { return Accessor(&skiplist); } + list_it_con find(const T &item) const { return accessor.find(item); } - const Accessor access() const { return Accessor(&skiplist); } + list_it find(const T &item) { return accessor.find(item); } - private: - list skiplist; + list_it_con find_or_larger(const T &item) const + { + return accessor.find_or_larger(item); + } + + list_it find_or_larger(const T &item) + { + return accessor.find_or_larger(item); + } + + bool contains(const T &item) const + { + return this->find(item) != this->end(); + } + + bool remove(const T &item) { return accessor.remove(item); } + }; + + Accessor access() { return Accessor(&skiplist); } + + const Accessor access() const { return Accessor(&skiplist); } + +private: + list skiplist; }; diff --git a/include/data_structures/concurrent/concurrent_set.hpp b/include/data_structures/concurrent/concurrent_set.hpp index 036463ad2..e0d749bee 100644 --- a/include/data_structures/concurrent/concurrent_set.hpp +++ b/include/data_structures/concurrent/concurrent_set.hpp @@ -37,6 +37,24 @@ public: list_it find(const T &item) { return accessor.find(item); } + template <class K> + list_it_con find_or_larger(const K &item) const + { + return accessor.find_or_larger(item); + } + + template <class K> + list_it find_or_larger(const K &item) + { + return accessor.find_or_larger(item); + } + + template <class K> + list_it_con cfind_or_larger(const K &item) + { + return accessor.template find_or_larger<list_it_con, K>(item); + } + bool contains(const T &item) const { return this->find(item) != this->end(); diff --git a/include/data_structures/concurrent/skiplist.hpp b/include/data_structures/concurrent/skiplist.hpp index bb7834a60..227c6559e 100644 --- a/include/data_structures/concurrent/skiplist.hpp +++ b/include/data_structures/concurrent/skiplist.hpp @@ -556,6 +556,19 @@ public: return skiplist->find(item); } + template <class K> + ConstIterator find_or_larger(const K &item) const + { + return static_cast<const SkipList &>(*skiplist).find_or_larger( + item); + } + + template <class It, class K> + It find_or_larger(const K &item) + { + return skiplist->find_or_larger<It, K>(item); + } + template <class K> bool contains(const K &item) const { @@ -637,6 +650,18 @@ private: template <class It, class K> It find_node(const K &item) + { + auto it = find_or_larger<It, K>(item); + if (it.node == nullptr || item == *it) { + return std::move(it); + } else { + return It(); + } + } + + // Returns iterator on searched element or the first larger element. + template <class It, class K> + It find_or_larger(const K &item) { Node *node, *pred = header; int h = static_cast<int>(pred->height) - 1; @@ -647,7 +672,7 @@ private: } // if we overshoot at every layer, item doesn't exist - if (h < 0) return It(); + if (h < 0) return It(node); // the item is farther to the right, continue going right as long // as the key is greater than the current node's key diff --git a/include/database/db.hpp b/include/database/db.hpp index 028ed518e..f28a1189d 100644 --- a/include/database/db.hpp +++ b/include/database/db.hpp @@ -1,5 +1,9 @@ #pragma once +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" + +#include "storage/garbage/garbage.hpp" #include "storage/graph.hpp" #include "transactions/engine.hpp" @@ -14,9 +18,24 @@ public: Graph graph; tx::Engine tx_engine; + Garbage garbage; std::string &name(); + // INDEXES + + // TG - type group + // I - type of function I:const tx::Transaction& -> + // std::unique_ptr<IndexBase<TypeGroupVertex,std::nullptr_t>> + // G - type of collection (verrtex/edge) + template <class TG, class I, class G> + bool create_index_on_vertex_property_family(const char *name, G &coll, + I &create_index); + + // Removes index IndexHolder. True if there was index to remove. + template <class TG, class K> + bool remove_index(IndexHolder<TG, K> &ih); + private: std::string name_; }; diff --git a/include/database/db_accessor.hpp b/include/database/db_accessor.hpp index 6f6bb9fb6..3f9e8fa75 100644 --- a/include/database/db_accessor.hpp +++ b/include/database/db_accessor.hpp @@ -1,9 +1,8 @@ #pragma once #include "database/db_transaction.hpp" -#include "storage/vertex_accessor.hpp" +#include "storage/model/properties/property_family.hpp" #include "utils/border.hpp" -// #include "utils/iterator/iterator.hpp" #include "utils/option.hpp" namespace tx @@ -11,23 +10,29 @@ namespace tx class Transaction; } +class Label; +class EdgeType; + +using EdgePropertyFamily = PropertyFamily<TypeGroupEdge>; +using VertexPropertyFamily = PropertyFamily<TypeGroupVertex>; + /* * DbAccessor * -Guarantees that access to Vertex and Edge is possible only through -* Vertex::Accessor and Edge::Accessor. +* VertexAccessor and EdgeAccessor. * -Guarantees that changing Vertex and Edge is possible only using -* Vertex::Accessor returned by vertex_insert() method and -* Edge::Accessor returned by edge_insert() method. +* VertexAccessor returned by vertex_insert() method and +* EdgeAccessor returned by edge_insert() method. * -Offers CRUD for Vertex and Edge except iterating over all edges. * -* Vertex::Accessor +* VertexAccessor * By default Vertex::accessor is empty. Caller has to call fill() method * to fetch valid data and check it's return value. fill() method returns * true if there is valid data for current transaction false otherwise. * Only exception to this rule is vertex_insert() method in DbAccessor -* which returns by default filled Vertex::Accessor. +* which returns by default filled VertexAccessor. * -* Edge::Accessor +* EdgeAccessor * By default Edge::accessor is empty. Caller has to call fill() method * to * fetch valid data and check it's return value. fill() method returns @@ -35,7 +40,7 @@ class Transaction; * if there is valid data for current transaction false otherwise. * Only exception to this rule is edge_insert() method in DbAccessor * which -* returns by default filled Edge::Accessor. +* returns by default filled EdgeAccessor. */ class DbAccessor { @@ -43,22 +48,24 @@ class DbAccessor public: DbAccessor(Db &db); + DbAccessor(Db &db, tx::Transaction &t); + //*******************VERTEX METHODS auto vertex_access(); - Option<const Vertex::Accessor> vertex_find(const Id &id); + Option<const VertexAccessor> vertex_find(const Id &id); - // Creates new Vertex and returns filled Vertex::Accessor. - Vertex::Accessor vertex_insert(); + // Creates new Vertex and returns filled VertexAccessor. + VertexAccessor vertex_insert(); // ******************* EDGE METHODS - Option<const Edge::Accessor> edge_find(const Id &id); + Option<const EdgeAccessor> edge_find(const Id &id); - // Creates new Edge and returns filled Edge::Accessor. - Edge::Accessor edge_insert(Vertex::Accessor const &from, - Vertex::Accessor const &to); + // Creates new Edge and returns filled EdgeAccessor. + EdgeAccessor edge_insert(VertexAccessor const &from, + VertexAccessor const &to); // ******************* LABEL METHODS @@ -74,25 +81,37 @@ public: // ******************** PROPERTY METHODS - PropertyFamily &vertex_property_family_get(const std::string &name); + VertexPropertyFamily &vertex_property_family_get(const std::string &name); - PropertyFamily &edge_property_family_get(const std::string &name); + EdgePropertyFamily &edge_property_family_get(const std::string &name); // ******************** PROPERTY HELPER METHODS - PropertyFamily::PropertyType::PropertyFamilyKey + VertexPropertyFamily::PropertyType::PropertyFamilyKey vertex_property_key(const std::string &name, Type type); - PropertyFamily::PropertyType::PropertyFamilyKey + EdgePropertyFamily::PropertyType::PropertyFamilyKey edge_property_key(const std::string &name, Type type); + template <class T> + VertexPropertyFamily::PropertyType::PropertyTypeKey<T> + vertex_property_key(const std::string &name); + + template <class T> + EdgePropertyFamily::PropertyType::PropertyTypeKey<T> + edge_property_key(const std::string &name); + // ******************** TRANSACTION METHODS - void commit(); + // True if commit was successful, or false if transaction was aborted. + bool commit(); void abort(); private: + // TODO: make this friend generic for all indexes. template <class T, class K> friend class NonUniqueUnorderedIndex; + template <class T, class K> + friend class UniqueOrderedIndex; DbTransaction db_transaction; }; diff --git a/include/database/db_transaction.hpp b/include/database/db_transaction.hpp index a36dd1d18..07972557e 100644 --- a/include/database/db_transaction.hpp +++ b/include/database/db_transaction.hpp @@ -1,10 +1,13 @@ #pragma once +#include "storage/indexes/index_update.hpp" #include "transactions/transaction.hpp" class Db; class DbAccessor; +using index_updates_t = std::vector<IndexUpdate>; + // Inner structures local to transaction can hold ref to this structure and use // its methods. // Also serves as a barrier for calling methods defined public but meant for @@ -21,6 +24,21 @@ public: // This should provide cleaner hierarchy of operations on database. // For example cleaner. + // Updates indexes of Vertex/Edges in index_updates. True if indexes are + // updated successfully. False means that transaction failed. + bool update_indexes(); + + // Will update indexes for given element TG::record_t. Actual update happens + // with call update_indexes + template <class TG> + void to_update_index(typename TG::vlist_t *vlist, + typename TG::record_t *record) + { + index_updates.push_back(make_index_update(vlist, record)); + } + + index_updates_t index_updates; + tx::Transaction &trans; Db &db; diff --git a/include/mvcc/record.hpp b/include/mvcc/record.hpp index 2731c2b56..bc2e0ece2 100644 --- a/include/mvcc/record.hpp +++ b/include/mvcc/record.hpp @@ -3,14 +3,14 @@ #include <atomic> #include <iostream> -#include "transactions/transaction.hpp" #include "transactions/commit_log.hpp" #include "transactions/engine.hpp" +#include "transactions/transaction.hpp" -#include "mvcc/id.hpp" #include "mvcc/cre_exp.hpp" -#include "mvcc/version.hpp" #include "mvcc/hints.hpp" +#include "mvcc/id.hpp" +#include "mvcc/version.hpp" #include "storage/locking/record_lock.hpp" // the mvcc implementation used here is very much like postgresql's @@ -41,7 +41,7 @@ public: RecordLock lock; // check if this record is visible to the transaction t - bool visible(const tx::Transaction& t) + bool visible(const tx::Transaction &t) { // TODO check if the record was created by a transaction that has been // aborted. one might implement this by checking the hints in mvcc @@ -51,87 +51,95 @@ public: // if you think they're not, you're wrong, and you should think about it // again. i know, it happened to me. - return ((tx.cre() == t.id && // inserted by the current transaction - cmd.cre() <= t.cid && // before this command, and - (tx.exp() == Id(0) || // the row has not been deleted, or - (tx.exp() == t.id && // it was deleted by the current - // transaction - cmd.exp() >= t.cid))) // but not before this command, - || // or - (cre_committed(tx.cre(), t) && // the record was inserted by a - // committed transaction, and - (tx.exp() == Id(0) || // the record has not been deleted, or - (tx.exp() == t.id && // the row is being deleted by this - // transaction - cmd.exp() >= t.cid) || // but it's not deleted "yet", or - (tx.exp() != t.id && // the row was deleted by another - // transaction - !exp_committed(tx.exp(), t) // that has not been committed - )))); + return ((tx.cre() == t.id && // inserted by the current transaction + cmd.cre() <= t.cid && // before this command, and + (tx.exp() == Id(0) || // the row has not been deleted, or + (tx.exp() == t.id && // it was deleted by the current + // transaction + cmd.exp() >= t.cid))) // but not before this command, + || // or + (cre_committed(tx.cre(), t) && // the record was inserted by a + // committed transaction, and + (tx.exp() == Id(0) || // the record has not been deleted, or + (tx.exp() == t.id && // the row is being deleted by this + // transaction + cmd.exp() >= t.cid) || // but it's not deleted "yet", or + (tx.exp() != t.id && // the row was deleted by another + // transaction + !exp_committed(tx.exp(), t) // that has not been committed + )))); } - void mark_created(const tx::Transaction& t) + void mark_created(const tx::Transaction &t) { tx.cre(t.id); cmd.cre(t.cid); } - void mark_deleted(const tx::Transaction& t) + void mark_deleted(const tx::Transaction &t) { tx.exp(t.id); cmd.exp(t.cid); } - bool exp_committed(const Id& id, const tx::Transaction& t) + bool exp_committed(const Id &id, const tx::Transaction &t) { return committed(hints.exp, id, t); } - bool exp_committed(const tx::Transaction& t) + bool exp_committed(const tx::Transaction &t) { return committed(hints.exp, tx.exp(), t); } - bool cre_committed(const Id& id, const tx::Transaction& t) + bool cre_committed(const Id &id, const tx::Transaction &t) { return committed(hints.cre, id, t); } - bool cre_committed(const tx::Transaction& t) + bool cre_committed(const tx::Transaction &t) { return committed(hints.cre, tx.cre(), t); } + // TODO: Test this + // True if this record is visible for write. + bool is_visible_write(const tx::Transaction &t) + { + return (tx.cre() == t.id && // inserted by the current transaction + cmd.cre() <= t.cid && // before this command, and + (tx.exp() == Id(0) || // the row has not been deleted, or + (tx.exp() == t.id && // it was deleted by the current + // transaction + cmd.exp() >= t.cid))); // but not before this command, + } + protected: template <class U> - bool committed(U& hints, const Id& id, const tx::Transaction& t) + bool committed(U &hints, const Id &id, const tx::Transaction &t) { // you certainly can't see the transaction with id greater than yours // as that means it started after this transaction and if it committed, // it committed after this transaction had started. - if(id > t.id) - return false; + if (id > t.id) return false; + + // The creating transaction is still in progress (examine snapshot) + if (t.is_active(id)) return false; auto hint_bits = hints.load(); + // TODO: Validate if this position is valid for next if. // if hints are set, return if xid is committed - if(!hint_bits.is_unknown()) - return hint_bits.is_committed(); + if (!hint_bits.is_unknown()) return hint_bits.is_committed(); // if hints are not set: - // - the creating transaction is still in progress (examine snapshot) - if(t.snapshot.is_active(id)) - return false; - // - you are the first one to check since it ended, consult commit log auto info = t.engine.clog.fetch_info(id); - if(info.is_committed()) - return hints.set_committed(), true; + if (info.is_committed()) return hints.set_committed(), true; assert(info.is_aborted()); return hints.set_aborted(), false; } }; - } diff --git a/include/mvcc/version_list.hpp b/include/mvcc/version_list.hpp index e6126201f..29161fe27 100644 --- a/include/mvcc/version_list.hpp +++ b/include/mvcc/version_list.hpp @@ -103,13 +103,6 @@ public: T *update(T *record, tx::Transaction &t) { assert(record != nullptr); - // TODO: VALIDATE NEXT IF BLOCK - if (record->tx.cre() == t.id) { - // THEN ONLY THIS TRANSACTION CAN SEE THIS DATA WHICH MENS THAT IT - // CAN CHANGE IT. - return record; - } - lock_and_validate(record, t); auto updated = new T(); @@ -149,7 +142,6 @@ private: void lock_and_validate(T *record, tx::Transaction &t) { assert(record != nullptr); - assert(record == find(t)); // take a lock on this node t.take_lock(lock); @@ -167,9 +159,3 @@ private: RecordLock lock; }; } - -class Vertex; -class Edge; - -using VertexRecord = mvcc::VersionList<Vertex>; -// using EdgeRecord = mvcc::VersionList<Edge>; diff --git a/include/query_engine/hardcode/queries.hpp b/include/query_engine/hardcode/queries.hpp index 8513b82de..bab402ce0 100644 --- a/include/query_engine/hardcode/queries.hpp +++ b/include/query_engine/hardcode/queries.hpp @@ -6,10 +6,12 @@ #include "query_engine/query_stripper.hpp" #include "query_engine/util.hpp" #include "storage/indexes/impl/nonunique_unordered_index.cpp" +// #include "storage/model/properties/properties.cpp" #include "storage/model/properties/property.hpp" #include "storage/model/properties/property_family.hpp" #include "utils/command_line/arguments.hpp" #include "utils/iterator/iterator.hpp" +// #include "utils/utils.cpp" auto load_queries(Db &db) { diff --git a/include/query_engine/query_result.hpp b/include/query_engine/query_result.hpp index 992df23c0..6ed096daf 100644 --- a/include/query_engine/query_result.hpp +++ b/include/query_engine/query_result.hpp @@ -10,10 +10,11 @@ #include "storage/model/properties/properties.hpp" +template <class TG> struct ResultList { using sptr = std::shared_ptr<ResultList>; - using data_t = std::vector<const Properties *>; + using data_t = std::vector<const Properties<TG> *>; ResultList() = default; ResultList(ResultList &other) = delete; @@ -22,13 +23,15 @@ struct ResultList explicit operator bool() const { return data.size() > 0; } - std::vector<const Properties *> data; + std::vector<const Properties<TG> *> data; }; +template <class TG> struct QueryResult { using sptr = std::shared_ptr<QueryResult>; - using data_t = std::unordered_map<std::string, ResultList::sptr>; + using data_t = + std::unordered_map<std::string, typename ResultList<TG>::sptr>; QueryResult() = default; QueryResult(QueryResult &other) = delete; diff --git a/include/query_engine/util.hpp b/include/query_engine/util.hpp index 40b5b5a57..0d853b258 100644 --- a/include/query_engine/util.hpp +++ b/include/query_engine/util.hpp @@ -11,7 +11,8 @@ using std::cout; using std::endl; -void print_props(const Properties &properties); +template <class T> +void print_props(const Properties<T> &properties); #ifdef NDEBUG #define PRINT_PROPS(_) @@ -19,7 +20,8 @@ void print_props(const Properties &properties); #define PRINT_PROPS(_PROPS_) print_props(_PROPS_); #endif -void cout_properties(const Properties &properties); +template <class T> +void cout_properties(const Properties<T> &properties); void cout_property(const std::string &key, const Property &property); diff --git a/include/storage/edge.hpp b/include/storage/edge.hpp index 55a3ea182..4e87c4d7d 100644 --- a/include/storage/edge.hpp +++ b/include/storage/edge.hpp @@ -2,7 +2,7 @@ #include "mvcc/record.hpp" #include "storage/model/edge_model.hpp" -#include "storage/model/properties/traversers/jsonwriter.hpp" +// #include "storage/model/properties/traversers/jsonwriter.hpp" class Edge : public mvcc::Record<Edge> { @@ -10,14 +10,14 @@ public: class Accessor; Edge() = default; - Edge(const EdgeModel& data) : data(data) {} - Edge(EdgeModel&& data) : data(std::move(data)) {} + Edge(const EdgeModel &data) : data(data) {} + Edge(EdgeModel &&data) : data(std::move(data)) {} - Edge(const Edge&) = delete; - Edge(Edge&&) = delete; + Edge(const Edge &) = delete; + Edge(Edge &&) = delete; - Edge& operator=(const Edge&) = delete; - Edge& operator=(Edge&&) = delete; + Edge &operator=(const Edge &) = delete; + Edge &operator=(Edge &&) = delete; EdgeModel data; }; diff --git a/include/storage/edge_accessor.hpp b/include/storage/edge_accessor.hpp index f62328eae..72a4323d0 100644 --- a/include/storage/edge_accessor.hpp +++ b/include/storage/edge_accessor.hpp @@ -7,18 +7,23 @@ #include "utils/assert.hpp" #include "utils/reference_wrapper.hpp" +class EdgeType; +using edge_type_ref_t = ReferenceWrapper<const EdgeType>; + class Edges; -class Edge::Accessor : public RecordAccessor<Edge, Edge::Accessor, EdgeRecord> +class EdgeAccessor : public RecordAccessor<TypeGroupEdge, EdgeAccessor> { public: using RecordAccessor::RecordAccessor; + typedef Edge record_t; + typedef EdgeRecord record_list_t; void edge_type(edge_type_ref_t edge_type); edge_type_ref_t edge_type() const; - Vertex::Accessor from() const; + VertexAccessor from() const; - Vertex::Accessor to() const; + VertexAccessor to() const; }; diff --git a/include/storage/edge_record.hpp b/include/storage/edge_record.hpp index aa1471abe..50ce0a42d 100644 --- a/include/storage/edge_record.hpp +++ b/include/storage/edge_record.hpp @@ -3,6 +3,8 @@ #include "mvcc/version_list.hpp" #include "storage/edge.hpp" +class VertexRecord; + class EdgeRecord : public mvcc::VersionList<Edge> { public: diff --git a/include/storage/edge_type/edge_type.hpp b/include/storage/edge_type/edge_type.hpp index 2dd64262c..94367aea1 100644 --- a/include/storage/edge_type/edge_type.hpp +++ b/include/storage/edge_type/edge_type.hpp @@ -3,18 +3,30 @@ #include <ostream> #include <stdint.h> +#include "storage/edge.hpp" +#include "storage/edge_accessor.hpp" +#include "storage/indexes/impl/nonunique_unordered_index.hpp" +#include "storage/type_group_edge.hpp" #include "utils/char_str.hpp" #include "utils/reference_wrapper.hpp" #include "utils/total_ordering.hpp" +using EdgeTypeIndexRecord = IndexRecord<TypeGroupEdge, std::nullptr_t>; + class EdgeType : public TotalOrdering<EdgeType> { public: - EdgeType(); + using type_index_t = NonUniqueUnorderedIndex<TypeGroupEdge, std::nullptr_t>; + + EdgeType() = delete; + EdgeType(const std::string &id); EdgeType(const char *id); EdgeType(std::string &&id); + EdgeType(const EdgeType &) = delete; + EdgeType(EdgeType &&other) = default; + friend bool operator<(const EdgeType &lhs, const EdgeType &rhs); friend bool operator==(const EdgeType &lhs, const EdgeType &rhs); @@ -25,6 +37,8 @@ public: CharStr char_str() { return CharStr(&id[0]); } + std::unique_ptr<type_index_t> index; + private: std::string id; }; diff --git a/include/storage/edges.hpp b/include/storage/edges.hpp index 080d39836..664f6d545 100644 --- a/include/storage/edges.hpp +++ b/include/storage/edges.hpp @@ -1,25 +1,30 @@ #pragma once #include <string> + #include "data_structures/concurrent/concurrent_map.hpp" -#include "mvcc/version_list.hpp" -#include "storage/common.hpp" -#include "storage/edge_accessor.hpp" -#include "storage/model/properties/property_family.hpp" +#include "utils/counters/atomic_counter.hpp" #include "utils/option.hpp" +#include "storage/edge_record.hpp" +#include "storage/model/properties/property_family.hpp" + +class EdgeAccessor; +class DbTransaction; + +using EdgePropertyFamily = PropertyFamily<TypeGroupEdge>; + class Edges { - using prop_familys_t = ConcurrentMap<std::string, PropertyFamily *>; + using prop_familys_t = ConcurrentMap<std::string, EdgePropertyFamily *>; public: - Option<const Edge::Accessor> find(DbTransaction &t, const Id &id); + Option<const EdgeAccessor> find(DbTransaction &t, const Id &id); - // Creates new Edge and returns filled Edge::Accessor. - Edge::Accessor insert(DbTransaction &t, VertexRecord *from, - VertexRecord *to); + // Creates new Edge and returns filled EdgeAccessor. + EdgeAccessor insert(DbTransaction &t, VertexRecord *from, VertexRecord *to); - PropertyFamily &property_family_find_or_create(const std::string &name); + EdgePropertyFamily &property_family_find_or_create(const std::string &name); private: ConcurrentMap<uint64_t, EdgeRecord> edges; diff --git a/include/storage/garbage/delete_sensitive.hpp b/include/storage/garbage/delete_sensitive.hpp new file mode 100644 index 000000000..973bde9d2 --- /dev/null +++ b/include/storage/garbage/delete_sensitive.hpp @@ -0,0 +1,9 @@ +#pragma once + +// Base class for all classes which need to be safely disposed. Main usage is +// for garbage class operations. +class DeleteSensitive +{ +public: + virtual ~DeleteSensitive() {} +}; diff --git a/include/storage/garbage/garbage.hpp b/include/storage/garbage/garbage.hpp new file mode 100644 index 000000000..4712fe780 --- /dev/null +++ b/include/storage/garbage/garbage.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include "data_structures/concurrent/concurrent_list.hpp" +#include "mvcc/id.hpp" +#include "storage/garbage/delete_sensitive.hpp" +#include "transactions/snapshot.hpp" + +namespace tx +{ +class Engine; +} + +// Collection of delete sensitive data which need to be safely deleted. That +// meens that all transactions that may have pointer to it must finish before +// the sensitive data can be safely destroyed. +class Garbage +{ +public: + void dispose(tx::Snapshot<Id> &&snapshot, DeleteSensitive *data); + + // Cleaner thread shoul call this method every some time. Removes data which + // is + // safe to be deleted. + void clean(tx::Engine &engine); + +private: + List<std::pair<tx::Snapshot<Id>, DeleteSensitive *>> gar; +}; diff --git a/include/storage/indexes/impl/nonunique_unordered_index.hpp b/include/storage/indexes/impl/nonunique_unordered_index.hpp index 01bebb1c5..77c76fc75 100644 --- a/include/storage/indexes/impl/nonunique_unordered_index.hpp +++ b/include/storage/indexes/impl/nonunique_unordered_index.hpp @@ -1,26 +1,29 @@ #pragma once #include "storage/indexes/index_base.hpp" -#include "storage/indexes/index_record.hpp" +// #include "storage/indexes/index_record.hpp" #include "data_structures/concurrent/concurrent_list.hpp" -template <class T, class K> -class NonUniqueUnorderedIndex : public IndexBase<T, K> +template <class TG, class K> +class NonUniqueUnorderedIndex : public IndexBase<TG, K> { public: - typedef T value_type; - typedef K key_type; + // typedef T value_type; + // typedef K key_type; + // Created with the database NonUniqueUnorderedIndex(); + NonUniqueUnorderedIndex(tx::Transaction const &t); + // Insert's value. // nonunique => always succeds. - bool insert(IndexRecord<T, K> &&value) final; + bool insert(IndexRecord<TG, K> &&value) final; // Returns iterator which returns valid records in range. // ordered==None => doesn't guarantee any order of submitting records. - std::unique_ptr<IteratorBase<const typename T::Accessor>> + std::unique_ptr<IteratorBase<const typename TG::accessor_t>> for_range(DbAccessor &t, Border<K> from = Border<K>(), Border<K> to = Border<K>()) final; @@ -34,5 +37,5 @@ public: void clean(DbTransaction &) final; private: - List<IndexRecord<T, K>> list; + List<IndexRecord<TG, K>> list; }; diff --git a/include/storage/indexes/impl/unique_ordered_index.hpp b/include/storage/indexes/impl/unique_ordered_index.hpp new file mode 100644 index 000000000..69daeecf7 --- /dev/null +++ b/include/storage/indexes/impl/unique_ordered_index.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include "storage/indexes/index_base.hpp" + +#include "data_structures/concurrent/concurrent_set.hpp" + +template <class T, class K> +class UniqueOrderedIndex : public IndexBase<T, K> +{ +public: + // typedef T value_type; + // typedef K key_type; + + // Created with the database + UniqueOrderedIndex(Order order); + + UniqueOrderedIndex(Order order, tx::Transaction const &t); + + // Insert's value. + // nonunique => always succeds. + bool insert(IndexRecord<T, K> &&value) final; + + // Returns iterator which returns valid records in range. + // ordered==None => doesn't guarantee any order of submitting records. + std::unique_ptr<IteratorBase<const typename T::accessor_t>> + for_range(DbAccessor &t, Border<K> from = Border<K>(), + Border<K> to = Border<K>()) final; + + // Same as for_range just whith known returned iterator. + auto for_range_exact(DbAccessor &t, Border<K> from = Border<K>(), + Border<K> to = Border<K>()); + + // Removes for all transactions obsolete Records. + // Cleaner has to call this method when he decideds that it is time for + // cleaning. + void clean(DbTransaction &) final; + +private: + ConcurrentSet<IndexRecord<T, K>> set; +}; diff --git a/include/storage/indexes/index_base.hpp b/include/storage/indexes/index_base.hpp index dd5e01529..06c00046f 100644 --- a/include/storage/indexes/index_base.hpp +++ b/include/storage/indexes/index_base.hpp @@ -1,41 +1,45 @@ #pragma once -// #include "storage/indexes/index_record.hpp" +#include <atomic> #include <functional> #include <memory> +#include "mvcc/id.hpp" + +// #include "storage/indexes/index_record.hpp" +#include "storage/garbage/delete_sensitive.hpp" #include "utils/border.hpp" #include "utils/iterator/iterator_base.hpp" +#include "utils/order.hpp" + +template <class TG, class K> +class IndexRecord; class DbTransaction; class DbAccessor; - -template <class T, class K> -class IndexRecord; - -// Defines ordering of data -enum Order +namespace tx { - None = 0, - Ascending = 1, - Descending = 2, -}; +class Transaction; +} // Interface for all indexes. -// T type of record. +// TG type group // K type of key on which records are ordered -template <class T, class K> -class IndexBase +template <class TG, class K> +class IndexBase : public DeleteSensitive { public: - typedef T value_type; - typedef K key_type; + // typedef T value_type; + // typedef K key_type; - IndexBase(bool unique, Order order) : unique(unique), order(order) {} + // Created with the database + IndexBase(bool unique, Order order); + + IndexBase(bool unique, Order order, const tx::Transaction &t); // Insert's value. // unique => returns false if there is already valid equal value. // nonunique => always succeds. - virtual bool insert(IndexRecord<T, K> &&value) = 0; + virtual bool insert(IndexRecord<TG, K> &&value) = 0; // Returns iterator which returns valid filled records in range. // order==noe => doesn't guarantee any order of returned records. @@ -44,7 +48,7 @@ public: // order==Descending => guarantees order of returned records will be from // largest to smallest. // Range must be from<=to - virtual std::unique_ptr<IteratorBase<const typename T::Accessor>> + virtual std::unique_ptr<IteratorBase<const typename TG::accessor_t>> for_range(DbAccessor &, Border<K> from = Border<K>(), Border<K> to = Border<K>()) = 0; @@ -53,8 +57,23 @@ public: // cleaning. virtual void clean(DbTransaction &) = 0; + // Activates index for readers. + void activate(); + + // True if index is ready for reading. + bool can_read(); + + // True if transaction is obliged to insert T into index. + bool is_obliged_to_insert(const tx::Transaction &t); + // Are the records unique const bool unique; // Ordering of the records. const Order order; + +private: + // Id of transaction which created this index. + const Id created; + // Active state + std::atomic_bool active = {false}; }; diff --git a/include/storage/indexes/index_holder.hpp b/include/storage/indexes/index_holder.hpp new file mode 100644 index 000000000..e8dac0386 --- /dev/null +++ b/include/storage/indexes/index_holder.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include <atomic> + +#include "storage/indexes/index_base.hpp" +#include "utils/option.hpp" +#include "utils/option_ptr.hpp" + +namespace tx +{ +class Transaction; +} + +// Holds onde index which can be changed. +// TG - type group +// K - key of index_records +template <class TG, class K> +class IndexHolder +{ + +public: + IndexHolder() = default; + + IndexHolder(IndexHolder const &) = delete; + + IndexHolder(IndexHolder &&) = default; + + // Sets index for this property family. Treturns false if index is already + // present. + bool set_index(std::unique_ptr<IndexBase<TG, K>> inx); + + // Returns index for read only if it is present and it's valid for read. + OptionPtr<IndexBase<TG, K>> get_read() const; + + // Returns index for write only if it's present and transaction is + // responsibly for updating it. + OptionPtr<IndexBase<TG, K>> get_write(const tx::Transaction &t) const; + + // Removes index if it is given index. Caller is now responsable of + // disposing index in a safe way. + Option<std::unique_ptr<IndexBase<TG, K>>> + remove_index(IndexBase<TG, K> *index); + + // Caller is now responsable of disposing index in a safe way. + Option<std::unique_ptr<IndexBase<TG, K>>> remove_index(); + +private: + std::atomic<IndexBase<TG, K> *> index = {nullptr}; +}; diff --git a/include/storage/indexes/index_record.hpp b/include/storage/indexes/index_record.hpp index 70f05af71..8381f8292 100644 --- a/include/storage/indexes/index_record.hpp +++ b/include/storage/indexes/index_record.hpp @@ -1,37 +1,32 @@ #pragma once -#include "database/db_transaction.hpp" -#include "mvcc/version_list.hpp" +#include "utils/border.hpp" #include "utils/total_ordering.hpp" -// class DbTransaction; -// namespace tx -// { -// class Transaction; -// } +namespace tx +{ +class Transaction; +} +class DbTransaction; -// T type of record. +// TG type group // K key on which record is ordered. -template <class T, class K> -class IndexRecord : public TotalOrdering<IndexRecord<T, K>> +template <class TG, class K> +class IndexRecord : public TotalOrdering<IndexRecord<TG, K>>, + public TotalOrdering<Border<K>, IndexRecord<TG, K>> { public: - using vlist_t = mvcc::VersionList<T>; - IndexRecord() = default; - IndexRecord(K key, T *record, vlist_t *vlist) - : key(std::move(key)), record(record), vlist(vlist) - { - assert(record != nullptr); - assert(vlist != nullptr); - } + IndexRecord(K key, typename TG::record_t *record, + typename TG::vlist_t *vlist); friend bool operator<(const IndexRecord &lhs, const IndexRecord &rhs) { - return lhs.key < rhs.key || - (lhs.key == rhs.key && lhs.vlist == rhs.vlist && - lhs.record < rhs.record); + return (lhs.key < rhs.key || + (lhs.key == rhs.key && lhs.vlist == rhs.vlist && + lhs.record < rhs.record)) ^ + lhs.descending; } friend bool operator==(const IndexRecord &lhs, const IndexRecord &rhs) @@ -40,28 +35,29 @@ public: (lhs.vlist != rhs.vlist || lhs.record == rhs.record); } - bool empty() const { return record == nullptr; } - - bool is_valid(tx::Transaction &t) const + friend bool operator<(const Border<K> &lhs, const IndexRecord &rhs) { - assert(!empty()); - return record == vlist->find(t); + return lhs < rhs.key; } - const auto access(DbTransaction &db) const + friend bool operator==(const Border<K> &lhs, const IndexRecord &rhs) { - return T::Accessor::create(record, vlist, db); + return lhs == rhs.key; } + // Will change ordering of record to descending. + void set_descending(); + + bool empty() const; + + bool is_valid(tx::Transaction &t) const; + + const auto access(DbTransaction &db) const; + const K key; private: - T *const record{nullptr}; - vlist_t *const vlist{nullptr}; + bool descending = false; + typename TG::record_t *const record{nullptr}; + typename TG::vlist_t *const vlist{nullptr}; }; - -template <class K> -using VertexIndexRecord = IndexRecord<Vertex, K>; - -template <class K> -using EdgeIndexRecord = IndexRecord<Edge, K>; diff --git a/include/storage/indexes/index_update.hpp b/include/storage/indexes/index_update.hpp new file mode 100644 index 000000000..e19650960 --- /dev/null +++ b/include/storage/indexes/index_update.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include "storage/indexes/index_record.hpp" +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" + +struct IndexUpdateEdge +{ + EdgeRecord *vlist; + Edge *record; +}; + +struct IndexUpdateVertex +{ + VertexRecord *vlist; + Vertex *record; +}; + +struct IndexUpdate +{ + enum + { + EDGE, + VERTEX + } tag; + + union + { + IndexUpdateEdge e; + IndexUpdateVertex v; + }; +}; + +template <class T, class V> +IndexUpdate make_index_update(V *vlist, T *record); + +template <> +IndexUpdate make_index_update(EdgeRecord *vlist, Edge *record); + +template <> +IndexUpdate make_index_update(VertexRecord *vlist, Vertex *record); diff --git a/include/storage/label/label.hpp b/include/storage/label/label.hpp index 441ff5ab2..12dc59f1b 100644 --- a/include/storage/label/label.hpp +++ b/include/storage/label/label.hpp @@ -9,13 +9,16 @@ #include "utils/char_str.hpp" #include "utils/reference_wrapper.hpp" #include "utils/total_ordering.hpp" +// #include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" -using LabelIndexRecord = VertexIndexRecord<std::nullptr_t>; +using LabelIndexRecord = IndexRecord<TypeGroupVertex, std::nullptr_t>; class Label : public TotalOrdering<Label>, TotalOrdering<CharStr, Label> { public: - using label_index_t = NonUniqueUnorderedIndex<Vertex, std::nullptr_t>; + using label_index_t = + NonUniqueUnorderedIndex<TypeGroupVertex, std::nullptr_t>; Label() = delete; @@ -36,10 +39,10 @@ public: operator const std::string &() const; - std::unique_ptr<label_index_t> index; - CharStr char_str() const { return CharStr(name.c_str()); } + std::unique_ptr<label_index_t> index; + private: std::string name; }; diff --git a/include/storage/model/edge_map.hpp b/include/storage/model/edge_map.hpp index e8d109cc5..5460e3e18 100644 --- a/include/storage/model/edge_map.hpp +++ b/include/storage/model/edge_map.hpp @@ -1,7 +1,8 @@ #pragma once #include "data_structures/map/rh_hashmultimap.hpp" -#include "mvcc/version_list.hpp" +#include "storage/edge_record.hpp" +// #include "storage/vertex_record.hpp" class EdgeMap { diff --git a/include/storage/model/edge_model.hpp b/include/storage/model/edge_model.hpp index a00754934..2d3be107e 100644 --- a/include/storage/model/edge_model.hpp +++ b/include/storage/model/edge_model.hpp @@ -1,10 +1,13 @@ #pragma once -#include "mvcc/version_list.hpp" +// #include "mvcc/version_list.hpp" +// #include "storage/edge_type/edge_type.hpp" #include "storage/model/property_model.hpp" -#include "storage/edge_type/edge_type.hpp" +#include "storage/type_group_edge.hpp" -class EdgeModel : public PropertyModel +class EdgeType; + +class EdgeModel : public PropertyModel<TypeGroupEdge> { public: // TODO: here should be the reference diff --git a/include/storage/model/properties/properties.hpp b/include/storage/model/properties/properties.hpp index 0d2554417..2e55fcbf6 100644 --- a/include/storage/model/properties/properties.hpp +++ b/include/storage/model/properties/properties.hpp @@ -6,16 +6,26 @@ #include "storage/model/properties/property_family.hpp" #include "utils/option.hpp" -using prop_key_t = PropertyFamily::PropertyType::PropertyFamilyKey; +template <class TG> +using prop_key_t = typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey; -template <class T> -using type_key_t = PropertyFamily::PropertyType::PropertyTypeKey<T>; +template <class TG, class T> +using type_key_t = + typename PropertyFamily<TG>::PropertyType::template PropertyTypeKey<T>; +template <class TG> class Properties { public: using sptr = std::shared_ptr<Properties>; + using prop_key_t = + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey; + + template <class T> + using type_key_t = + typename PropertyFamily<TG>::PropertyType::template PropertyTypeKey<T>; + auto begin() const { return props.begin(); } auto cbegin() const { return props.cbegin(); } @@ -24,7 +34,7 @@ public: size_t size() const { return props.size(); } - const Property &at(PropertyFamily &key) const; + const Property &at(PropertyFamily<TG> &key) const; const Property &at(prop_key_t &key) const; @@ -38,7 +48,7 @@ public: void clear(prop_key_t &key); - void clear(PropertyFamily &key); + void clear(PropertyFamily<TG> &key); template <class Handler> void accept(Handler &handler) const @@ -58,6 +68,6 @@ public: private: using props_t = - std::unordered_map<prop_key_t, Property::sptr, PropertyHash>; + std::unordered_map<prop_key_t, Property::sptr, PropertyHash<TG>>; props_t props; }; diff --git a/include/storage/model/properties/property_family.hpp b/include/storage/model/properties/property_family.hpp index 653c78582..b71db0c7d 100644 --- a/include/storage/model/properties/property_family.hpp +++ b/include/storage/model/properties/property_family.hpp @@ -1,15 +1,21 @@ #pragma once #include <memory> + #include "data_structures/concurrent/concurrent_map.hpp" +#include "storage/indexes/index_holder.hpp" #include "storage/model/properties/flags.hpp" #include "utils/option.hpp" #include "utils/total_ordering.hpp" #include "utils/underlying_cast.hpp" +// #include "storage/indexes/index_base.hpp" + // Family of properties with the same name but different types. // Ordered on name. -class PropertyFamily : public TotalOrdering<PropertyFamily> +// TG - group of types Edge/Vertex +template <class TG> +class PropertyFamily : public TotalOrdering<PropertyFamily<TG>> { friend class PropertyType; friend class PropertyFamilyKey; @@ -175,6 +181,8 @@ public: friend bool operator==(const PropertyFamily &lhs, const PropertyFamily &rhs); + IndexHolder<TG, std::nullptr_t> index; + private: const std::string name_v; // This is exclusivly for getNull method. @@ -184,11 +192,13 @@ private: ConcurrentMap<Type, std::unique_ptr<PropertyType>> types; }; +template <class TG> class PropertyHash { public: - size_t - operator()(PropertyFamily::PropertyType::PropertyFamilyKey const &key) const + size_t operator()( + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey const &key) + const { return (std::hash<const void *>()((const void *)(&(key.get_family()))) + 7) * diff --git a/include/storage/model/properties/traversers/consolewriter.hpp b/include/storage/model/properties/traversers/consolewriter.hpp index dedc5eb4d..e81b1c534 100644 --- a/include/storage/model/properties/traversers/consolewriter.hpp +++ b/include/storage/model/properties/traversers/consolewriter.hpp @@ -4,6 +4,8 @@ #include "storage/model/properties/handler.hpp" #include "storage/model/properties/properties.hpp" +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" using std::cout; using std::endl; @@ -13,7 +15,24 @@ class ConsoleWriter public: ConsoleWriter() {} - void handle(const prop_key_t &key, const Property &value) + void handle(const typename PropertyFamily< + TypeGroupEdge>::PropertyType::PropertyFamilyKey &key, + const Property &value) + { + handle<TypeGroupEdge>(key, value); + } + + void handle(const typename PropertyFamily< + TypeGroupVertex>::PropertyType::PropertyFamilyKey &key, + const Property &value) + { + handle<TypeGroupVertex>(key, value); + } + + template <class T> + void handle( + const typename PropertyFamily<T>::PropertyType::PropertyFamilyKey &key, + const Property &value) { cout << "KEY: " << key.family_name() << "; VALUE: "; diff --git a/include/storage/model/properties/traversers/jsonwriter.hpp b/include/storage/model/properties/traversers/jsonwriter.hpp index 3daabcbef..449026783 100644 --- a/include/storage/model/properties/traversers/jsonwriter.hpp +++ b/include/storage/model/properties/traversers/jsonwriter.hpp @@ -2,6 +2,8 @@ #include "storage/model/properties/handler.hpp" #include "storage/model/properties/properties.hpp" +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" template <class Buffer> struct JsonWriter @@ -9,7 +11,24 @@ struct JsonWriter public: JsonWriter(Buffer &buffer) : buffer(buffer) { buffer << '{'; }; - void handle(const prop_key_t &key, const Property &value) + void handle(const typename PropertyFamily< + TypeGroupEdge>::PropertyType::PropertyFamilyKey &key, + const Property &value) + { + handle<TypeGroupEdge>(key, value); + } + + void handle(const typename PropertyFamily< + TypeGroupVertex>::PropertyType::PropertyFamilyKey &key, + const Property &value) + { + handle<TypeGroupVertex>(key, value); + } + + template <class T> + void handle( + const typename PropertyFamily<T>::PropertyType::PropertyFamilyKey &key, + const Property &value) { if (!first) buffer << ','; diff --git a/include/storage/model/property_model.hpp b/include/storage/model/property_model.hpp index d6c7bb193..466446723 100644 --- a/include/storage/model/property_model.hpp +++ b/include/storage/model/property_model.hpp @@ -2,8 +2,9 @@ #include "storage/model/properties/properties.hpp" +template <class TG> class PropertyModel { public: - Properties props; + Properties<TG> props; }; diff --git a/include/storage/model/vertex_model.hpp b/include/storage/model/vertex_model.hpp index 9293d0c51..088fbd549 100644 --- a/include/storage/model/vertex_model.hpp +++ b/include/storage/model/vertex_model.hpp @@ -5,7 +5,7 @@ #include "storage/model/edge_map.hpp" #include "storage/model/property_model.hpp" -class VertexModel : public PropertyModel +class VertexModel : public PropertyModel<TypeGroupVertex> { public: EdgeList out; diff --git a/include/storage/record_accessor.hpp b/include/storage/record_accessor.hpp index c72137406..945a7d696 100644 --- a/include/storage/record_accessor.hpp +++ b/include/storage/record_accessor.hpp @@ -3,15 +3,18 @@ #include "database/db_transaction.hpp" #include "mvcc/version_list.hpp" #include "storage/indexes/index_record.hpp" +#include "storage/indexes/index_update.hpp" #include "storage/model/properties/properties.hpp" #include "storage/model/properties/property.hpp" #include "storage/model/properties/property_family.hpp" #include "transactions/transaction.hpp" -template <class T, class Derived, class vlist_t = mvcc::VersionList<T>> +template <class TG, class Derived> class RecordAccessor { friend DbAccessor; + using vlist_t = typename TG::vlist_t; + using T = typename TG::record_t; public: RecordAccessor(vlist_t *vlist, DbTransaction &db) : vlist(vlist), db(db) @@ -47,11 +50,27 @@ public: const Id &id() const { return vlist->id; } + // TODO: Test this Derived update() const { assert(!empty()); - return Derived(vlist->update(db.trans), vlist, db); + if (record->is_visible_write(db.trans)) { + // TODO: VALIDATE THIS BRANCH. THEN ONLY THIS TRANSACTION CAN SEE + // THIS DATA WHICH MEENS THAT IT CAN CHANGE IT. + return Derived(record, vlist, db); + + } else { + auto new_record = vlist->update(db.trans); + + // TODO: Validate that update of record in this accessor is correct. + const_cast<RecordAccessor *>(this)->record = new_record; + + // Add record to update index. + db.to_update_index<TG>(vlist, new_record); + + return Derived(new_record, vlist, db); + } } bool remove() const @@ -61,30 +80,36 @@ public: return vlist->remove(record, db.trans); } - const Property &at(PropertyFamily &key) const + const Property &at(PropertyFamily<TG> &key) const { return properties().at(key); } - const Property &at(prop_key_t &key) const { return properties().at(key); } + const Property &at(prop_key_t<TG> &key) const + { + return properties().at(key); + } template <class V> - auto at(type_key_t<V> &key) const; + auto at(type_key_t<TG, V> &key) const + { + return properties().template at<V>(key); + } template <class V, class... Args> - void set(type_key_t<V> &key, Args &&... args) + void set(type_key_t<TG, V> &key, Args &&... args) { properties().template set<V>(key, std::forward<Args>(args)...); } - void set(prop_key_t &key, Property::sptr value) + void set(prop_key_t<TG> &key, Property::sptr value) { properties().set(key, std::move(value)); } - void clear(prop_key_t &key) { properties().clear(key); } + void clear(prop_key_t<TG> &key) { properties().clear(key); } - void clear(PropertyFamily &key) { properties().clear(key); } + void clear(PropertyFamily<TG> &key) { properties().clear(key); } template <class Handler> void accept(Handler &handler) const @@ -92,7 +117,7 @@ public: properties().template accept<Handler>(handler); } - Properties &properties() const { return record->data.props; } + Properties<TG> &properties() const { return record->data.props; } explicit operator bool() const { return record != nullptr; } @@ -127,18 +152,6 @@ public: return !(a == b); } -protected: - IndexRecord<T, std::nullptr_t> create_index_record() - { - return create_index_record(std::nullptr_t()); - } - - template <class K> - IndexRecord<T, K> create_index_record(K &&key) - { - return IndexRecord<T, K>(std::move(key), record, vlist); - } - T *record{nullptr}; vlist_t *const vlist; DbTransaction &db; diff --git a/include/storage/type_group_edge.hpp b/include/storage/type_group_edge.hpp new file mode 100644 index 000000000..f0871f865 --- /dev/null +++ b/include/storage/type_group_edge.hpp @@ -0,0 +1,19 @@ +#pragma once + +class Edge; +class EdgeRecord; +class EdgeAccessor; + +// Types for Edge side of database. Firstly there exists need for knowing the +// type of object to be able to efficently use it. Dependant classes can +// templetaze over such type, but this is anoying and error prone if there are +// multiple such types over which it is necessary to templetaze. The idea is to +// unify groups of logicaly tyed types into one type. That way depending classes +// can template over that one type. +class TypeGroupEdge +{ +public: + using record_t = Edge; + using vlist_t = EdgeRecord; + using accessor_t = EdgeAccessor; +}; diff --git a/include/storage/type_group_vertex.hpp b/include/storage/type_group_vertex.hpp new file mode 100644 index 000000000..2c103ee78 --- /dev/null +++ b/include/storage/type_group_vertex.hpp @@ -0,0 +1,19 @@ +#pragma once + +class Vertex; +class VertexRecord; +class VertexAccessor; + +// Types for Vertex side of database. Firstly there exists need for knowing the +// type of object to be able to efficently use it. Dependant classes can +// templetaze over such type, but this is anoying and error prone if there are +// multiple such types over which it is necessary to templetaze. The idea is to +// unify groups of logicaly tyed types into one type. That way depending classes +// can template over that one type. +class TypeGroupVertex +{ +public: + using record_t = Vertex; + using vlist_t = VertexRecord; + using accessor_t = VertexAccessor; +}; diff --git a/include/storage/vertex_accessor.hpp b/include/storage/vertex_accessor.hpp index 71b65d0c1..4a90481e0 100644 --- a/include/storage/vertex_accessor.hpp +++ b/include/storage/vertex_accessor.hpp @@ -5,16 +5,12 @@ class Vertices; -class Vertex::Accessor : public RecordAccessor<Vertex, Vertex::Accessor> +class VertexAccessor : public RecordAccessor<TypeGroupVertex, VertexAccessor> { public: using RecordAccessor::RecordAccessor; - - static Vertex::Accessor create(Vertex *t, mvcc::VersionList<Vertex> *vlist, - DbTransaction &db) - { - return Vertex::Accessor(t, vlist, db); - } + typedef Vertex record_t; + typedef VertexRecord record_list_t; size_t out_degree() const; @@ -37,5 +33,5 @@ public: auto in() const; // True if there exists edge other->this - bool in_contains(Vertex::Accessor const &other) const; + bool in_contains(VertexAccessor const &other) const; }; diff --git a/include/storage/vertex_record.hpp b/include/storage/vertex_record.hpp new file mode 100644 index 000000000..72051795b --- /dev/null +++ b/include/storage/vertex_record.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include "mvcc/version_list.hpp" +#include "storage/vertex.hpp" + +class VertexRecord : public mvcc::VersionList<Vertex> +{ +public: + VertexRecord(Id id) : VersionList(id) {} + VertexRecord(const VersionList &) = delete; + VertexRecord(VersionList &&other) : VersionList(std::move(other)) {} +}; diff --git a/include/storage/vertices.hpp b/include/storage/vertices.hpp index c90293c7a..a10f43b08 100644 --- a/include/storage/vertices.hpp +++ b/include/storage/vertices.hpp @@ -1,31 +1,35 @@ #pragma once -#include <memory> #include <string> + #include "data_structures/concurrent/concurrent_map.hpp" -#include "database/db_transaction.hpp" -#include "storage/common.hpp" -#include "storage/model/properties/property_family.hpp" -#include "storage/vertex_accessor.hpp" +#include "utils/counters/atomic_counter.hpp" #include "utils/option.hpp" +#include "storage/model/properties/property_family.hpp" +#include "storage/vertex_record.hpp" + class DbTransaction; +class VertexAccessor; + +using VertexPropertyFamily = PropertyFamily<TypeGroupVertex>; class Vertices { public: using vertices_t = ConcurrentMap<uint64_t, VertexRecord>; using prop_familys_t = - ConcurrentMap<std::string, std::unique_ptr<PropertyFamily>>; + ConcurrentMap<std::string, std::unique_ptr<VertexPropertyFamily>>; vertices_t::Accessor access(); - Option<const Vertex::Accessor> find(DbTransaction &t, const Id &id); + Option<const VertexAccessor> find(DbTransaction &t, const Id &id); - // Creates new Vertex and returns filled Vertex::Accessor. - Vertex::Accessor insert(DbTransaction &t); + // Creates new Vertex and returns filled VertexAccessor. + VertexAccessor insert(DbTransaction &t); - PropertyFamily &property_family_find_or_create(const std::string &name); + VertexPropertyFamily & + property_family_find_or_create(const std::string &name); private: vertices_t vertices; diff --git a/include/transactions/commit_log.hpp b/include/transactions/commit_log.hpp index 9f0608668..cc043ac6a 100644 --- a/include/transactions/commit_log.hpp +++ b/include/transactions/commit_log.hpp @@ -48,6 +48,9 @@ public: void set_aborted(const Id &id) { log.set(2 * id + 1); } private: + // TODO: Searching the log will take more and more time the more and more + // transactoins are done. This could be awerted if DynamicBitset is changed + // to point to largest chunk instead of the smallest. DynamicBitset<uint8_t, 32768> log; }; } diff --git a/include/transactions/engine.hpp b/include/transactions/engine.hpp index 2fb9d95d7..0348f80de 100644 --- a/include/transactions/engine.hpp +++ b/include/transactions/engine.hpp @@ -26,7 +26,10 @@ public: Engine() : counter(1) {} - Transaction &begin() + // Begins transaction and runs given functions in same atomic step. + // Functions will be given Transaction& + template <class... F> + Transaction &begin(F... fun) { auto guard = this->acquire_unique(); @@ -36,6 +39,8 @@ public: active.insert(id); store.put(id, t); + call(*t, fun...); + return *t; } @@ -53,6 +58,14 @@ public: return *t; } + // Returns copy of current snapshot + Snapshot<Id> snapshot() + { + auto guard = this->acquire_unique(); + + return active; + } + void commit(const Transaction &t) { auto guard = this->acquire_unique(); @@ -92,6 +105,21 @@ public: CommitLog clog; private: + template <class T, class... F> + void call(Transaction &t, T fun, F... funs) + { + call(t, fun); + call(t, funs...); + } + + template <class T> + void call(Transaction &t, T fun) + { + fun(t); + } + + void call(Transaction &t) {} + void finalize(const Transaction &t) { active.remove(t.id); diff --git a/include/transactions/snapshot.hpp b/include/transactions/snapshot.hpp index 8c558b3e6..04168a959 100644 --- a/include/transactions/snapshot.hpp +++ b/include/transactions/snapshot.hpp @@ -1,7 +1,7 @@ #pragma once -#include <vector> #include <algorithm> +#include <vector> namespace tx { @@ -14,45 +14,31 @@ public: Snapshot(std::vector<id_t> active) : active(std::move(active)) {} - Snapshot(const Snapshot& other) - { - active = other.active; - } + Snapshot(const Snapshot &other) { active = other.active; } - Snapshot(Snapshot&& other) - { - active = std::move(other.active); - } + Snapshot(Snapshot &&other) { active = std::move(other.active); } bool is_active(id_t xid) const { return std::binary_search(active.begin(), active.end(), xid); } - void insert(const id_t& id) - { - active.push_back(id); - } + void insert(const id_t &id) { active.push_back(id); } - void remove(const id_t& id) + void remove(const id_t &id) { // remove transaction from the active transactions list auto last = std::remove(active.begin(), active.end(), id); active.erase(last, active.end()); } - const id_t& front() - { - return active.front(); - } + const id_t &front() { return active.front(); } - size_t size() - { - return active.size(); - } + const id_t &back() { return active.back(); } + + size_t size() { return active.size(); } private: std::vector<id_t> active; }; - } diff --git a/include/transactions/transaction.hpp b/include/transactions/transaction.hpp index f287282df..7c618aadf 100644 --- a/include/transactions/transaction.hpp +++ b/include/transactions/transaction.hpp @@ -28,8 +28,13 @@ public: // index of the current command in the current transaction; uint8_t cid; // a snapshot of currently active transactions - const Snapshot<Id> snapshot; + // Blocks until all transactions from snapshot finish. After this method, + // snapshot will be empty. + void wait_for_active(); + + // True if id is in snapshot. + bool is_active(const Id &id) const; void take_lock(RecordLock &lock); void commit(); void abort(); @@ -37,6 +42,7 @@ public: Engine &engine; private: + Snapshot<Id> snapshot; LockStore<RecordLock> locks; }; } diff --git a/include/utils/border.hpp b/include/utils/border.hpp index 641ee1d29..16c9555bb 100644 --- a/include/utils/border.hpp +++ b/include/utils/border.hpp @@ -1,6 +1,7 @@ #pragma once #include "utils/option.hpp" +#include "utils/total_ordering.hpp" // Defines Including as [ and Excluding < for ranges. enum BorderType @@ -9,6 +10,7 @@ enum BorderType Excluding = 1, }; +// If key is not present he is both the largest and the smallest of the keys. template <class T> class Border { @@ -26,22 +28,60 @@ public: Border &operator=(Border &&other) = default; Border &operator=(Border &other) = default; - // true if no border or this > key or this >= key depends on border type. - bool operator>(const T &other) const + friend bool operator<(const Border<T> &a, const T &other) { - return !key.is_present() || key.get() > other || - (type == Including && key.get() == other); + return !a.key.is_present() || + (a.type == Excluding && a.key.get() <= other) || + (a.type == Including && a.key.get() < other); } - // true if no border or this < key or this <= key depends on border type. - bool operator<(const T &other) const + friend bool operator==(const Border<T> &a, const T &other) { - return !key.is_present() || key.get() < other || - (type == Including && key.get() == other); + return a.type == Including && a.key.is_present() && + a.key.get() == other; } + friend bool operator>(const Border<T> &a, const T &other) + { + return !a.key.is_present() || + (a.type == Excluding && a.key.get() >= other) || + (a.type == Including && a.key.get() > other); + } + + friend bool operator!=(const Border<T> &a, const T &b) { return !(a == b); } + + friend bool operator<=(const Border<T> &a, const T &b) + { + return a < b || a == b; + } + + friend bool operator>=(const Border<T> &a, const T &b) + { + return a > b || a == b; + } + + // // true if no border or this > key or this >= key depends on border type. + // bool operator>(const T &other) const + // { + // return !key.is_present() || key.get() > other || + // (type == Including && key.get() == other); + // } + // + // // true if this border is inclusive and key is present and key == other. + // bool operator==(const T &other) const + // { + // return type == Including && key.is_present() && key.get() == other; + // } + // + // // true if no border or this < key or this <= key depends on border type. + // bool operator<(const T &other) const + // { + // return !key.is_present() || key.get() < other || + // (type == Including && key.get() == other); + // } + Option<T> key; - const BorderType type; + BorderType type; }; template <class T> diff --git a/include/utils/option_ptr.hpp b/include/utils/option_ptr.hpp index b3fb0f550..a57502788 100644 --- a/include/utils/option_ptr.hpp +++ b/include/utils/option_ptr.hpp @@ -18,3 +18,9 @@ public: private: T *ptr = nullptr; }; + +template <class T> +auto make_option_ptr(T *t) +{ + return OptionPtr<T>(t); +} diff --git a/include/utils/order.hpp b/include/utils/order.hpp new file mode 100644 index 000000000..645609320 --- /dev/null +++ b/include/utils/order.hpp @@ -0,0 +1,9 @@ +#pragma once + +// Defines ordering of data +enum Order +{ + None = 0, + Ascending = 1, + Descending = 2, +}; diff --git a/poc/astar.cpp b/poc/astar.cpp index c14084fa2..c4a6a09aa 100644 --- a/poc/astar.cpp +++ b/poc/astar.cpp @@ -28,7 +28,7 @@ const int max_score = 1000000; using namespace std; -typedef Vertex::Accessor VertexAccessor; +typedef VertexAccessor VertexAccessor; void add_scores(Db &db); @@ -36,17 +36,18 @@ class Node { public: Node *parent = {nullptr}; - type_key_t<Double> tkey; + type_key_t<TypeGroupVertex, Double> tkey; double cost; int depth = {0}; VertexAccessor vacc; - Node(VertexAccessor vacc, double cost, type_key_t<Double> tkey) + Node(VertexAccessor vacc, double cost, + type_key_t<TypeGroupVertex, Double> tkey) : cost(cost), vacc(vacc), tkey(tkey) { } Node(VertexAccessor vacc, double cost, Node *parent, - type_key_t<Double> tkey) + type_key_t<TypeGroupVertex, Double> tkey) : cost(cost), vacc(vacc), parent(parent), depth(parent->depth + 1), tkey(tkey) { @@ -84,27 +85,27 @@ void found_result(Node *res) } } -double calc_heuristic_cost_dummy(type_key_t<Double> tkey, Edge::Accessor &edge, - Vertex::Accessor &vertex) +double calc_heuristic_cost_dummy(type_key_t<TypeGroupVertex, Double> tkey, + EdgeAccessor &edge, VertexAccessor &vertex) { assert(!vertex.empty()); return 1 - *vertex.at(tkey).get(); } -typedef bool (*EdgeFilter)(DbAccessor &t, Edge::Accessor &, Node *before); -typedef bool (*VertexFilter)(DbAccessor &t, Vertex::Accessor &, Node *before); +typedef bool (*EdgeFilter)(DbAccessor &t, EdgeAccessor &, Node *before); +typedef bool (*VertexFilter)(DbAccessor &t, VertexAccessor &, Node *before); -bool edge_filter_dummy(DbAccessor &t, Edge::Accessor &e, Node *before) +bool edge_filter_dummy(DbAccessor &t, EdgeAccessor &e, Node *before) { return true; } -bool vertex_filter_dummy(DbAccessor &t, Vertex::Accessor &va, Node *before) +bool vertex_filter_dummy(DbAccessor &t, VertexAccessor &va, Node *before) { return va.fill(); } -bool vertex_filter_contained_dummy(DbAccessor &t, Vertex::Accessor &v, +bool vertex_filter_contained_dummy(DbAccessor &t, VertexAccessor &v, Node *before) { if (v.fill()) { @@ -128,7 +129,7 @@ bool vertex_filter_contained_dummy(DbAccessor &t, Vertex::Accessor &v, return false; } -bool vertex_filter_contained(DbAccessor &t, Vertex::Accessor &v, Node *before) +bool vertex_filter_contained(DbAccessor &t, VertexAccessor &v, Node *before) { if (v.fill()) { bool found; @@ -146,17 +147,18 @@ bool vertex_filter_contained(DbAccessor &t, Vertex::Accessor &v, Node *before) // Vertex filter ima max_depth funkcija te edge filter ima max_depth funkcija. // Jedan za svaku dubinu. // Filtri vracaju true ako element zadovoljava uvjete. -auto a_star(Db &db, int64_t sys_id_start, uint max_depth, EdgeFilter e_filter[], - VertexFilter v_filter[], - double (*calc_heuristic_cost)(type_key_t<Double> tkey, - Edge::Accessor &edge, - Vertex::Accessor &vertex), - int limit) +auto a_star( + Db &db, int64_t sys_id_start, uint max_depth, EdgeFilter e_filter[], + VertexFilter v_filter[], + double (*calc_heuristic_cost)(type_key_t<TypeGroupVertex, Double> tkey, + EdgeAccessor &edge, VertexAccessor &vertex), + int limit) { DbAccessor t(db); - type_key_t<Double> tkey = t.vertex_property_family_get("score") - .get(Flags::Double) - .type_key<Double>(); + type_key_t<TypeGroupVertex, Double> tkey = + t.vertex_property_family_get("score") + .get(Flags::Double) + .type_key<Double>(); auto best_found = new std::map<Id, Score>[max_depth]; diff --git a/poc/tool.cpp b/poc/tool.cpp index 616db34f2..1f16f16c9 100644 --- a/poc/tool.cpp +++ b/poc/tool.cpp @@ -20,7 +20,7 @@ using namespace std; // generation. template <class C> -void fill_to_fill(Edge::Accessor &e, const EdgeType &type, C &&consumer) +void fill_to_fill(EdgeAccessor &e, const EdgeType &type, C &&consumer) { if (e.fill() && e.edge_type() == type) { auto to = e.to(); @@ -31,7 +31,7 @@ void fill_to_fill(Edge::Accessor &e, const EdgeType &type, C &&consumer) } template <class C> -void fill_from_fill(Edge::Accessor &e, const EdgeType &type, C &&consumer) +void fill_from_fill(EdgeAccessor &e, const EdgeType &type, C &&consumer) { if (e.fill() && e.edge_type() == type) { auto from = e.from(); @@ -42,7 +42,7 @@ void fill_from_fill(Edge::Accessor &e, const EdgeType &type, C &&consumer) } template <class C> -void fill_to_fill(Edge::Accessor &e, C &&consumer) +void fill_to_fill(EdgeAccessor &e, C &&consumer) { if (e.fill()) { auto to = e.to(); @@ -53,7 +53,7 @@ void fill_to_fill(Edge::Accessor &e, C &&consumer) } template <class C> -void to_fill(Edge::Accessor &e, C &&consumer) +void to_fill(EdgeAccessor &e, C &&consumer) { auto to = e.to(); if (to.fill()) { @@ -62,7 +62,7 @@ void to_fill(Edge::Accessor &e, C &&consumer) } template <class C> -void to_fill(Edge::Accessor &e, const Label &label, C &&consumer) +void to_fill(EdgeAccessor &e, const Label &label, C &&consumer) { auto to = e.to(); if (to.fill() && to.has_label(label)) { @@ -71,7 +71,7 @@ void to_fill(Edge::Accessor &e, const Label &label, C &&consumer) } template <class C> -void to_fill(Edge::Accessor &e, const EdgeType &type, const Label &label, +void to_fill(EdgeAccessor &e, const EdgeType &type, const Label &label, C &&consumer) { if (e.edge_type() == type) { @@ -83,7 +83,7 @@ void to_fill(Edge::Accessor &e, const EdgeType &type, const Label &label, } template <class C> -void from_fill(Edge::Accessor &e, const EdgeType &type, C &&consumer) +void from_fill(EdgeAccessor &e, const EdgeType &type, C &&consumer) { if (e.edge_type() == type) { auto from = e.from(); @@ -94,7 +94,7 @@ void from_fill(Edge::Accessor &e, const EdgeType &type, C &&consumer) } template <class C> -void fill_from_fill(Edge::Accessor &e, C &&consumer) +void fill_from_fill(EdgeAccessor &e, C &&consumer) { if (e.fill()) { auto from = e.from(); @@ -144,10 +144,10 @@ void find_fill(I iter, C &&consumer) } } -void fill_with_bt(unordered_map<string, double> &values, Vertex::Accessor &com, - double weight, - PropertyFamily::PropertyType::PropertyTypeKey<ArrayString> - &prop_vertex_business_types) +void fill_with_bt( + unordered_map<string, double> &values, VertexAccessor &com, double weight, + VertexPropertyFamily::PropertyType::PropertyTypeKey<ArrayString> + &prop_vertex_business_types) { auto bus_t = com.at(prop_vertex_business_types); if (bus_t.is_present()) { @@ -158,8 +158,8 @@ void fill_with_bt(unordered_map<string, double> &values, Vertex::Accessor &com, } void oportunity_employe_company( - Vertex::Accessor &va, unordered_map<string, double> &values, double weight, - PropertyFamily::PropertyType::PropertyTypeKey<ArrayString> + VertexAccessor &va, unordered_map<string, double> &values, double weight, + VertexPropertyFamily::PropertyType::PropertyTypeKey<ArrayString> &prop_vertex_business_types, const EdgeType &type_created, const EdgeType &type_works_in, const Label &label_company) @@ -192,16 +192,17 @@ auto query(DbAccessor &t, const Id &start_id) const Label &label_company = t.label_find_or_create("Company"); const Label &label_opportunuty = t.label_find_or_create("Opportunity"); - auto type_works_in = t.type_find_or_create("Works_In"); - auto type_reached_to = t.type_find_or_create("Reached_To"); - auto type_partnered_with = t.type_find_or_create("Partnered_With"); - auto type_interested_in = t.type_find_or_create("Interested_In"); - auto type_viewed = t.type_find_or_create("Viewed"); - auto type_has_match = t.type_find_or_create("Has_Match"); - auto type_searched_and_clicked = + const EdgeType &type_works_in = t.type_find_or_create("Works_In"); + const EdgeType &type_reached_to = t.type_find_or_create("Reached_To"); + const EdgeType &type_partnered_with = + t.type_find_or_create("Partnered_With"); + const EdgeType &type_interested_in = t.type_find_or_create("Interested_In"); + const EdgeType &type_viewed = t.type_find_or_create("Viewed"); + const EdgeType &type_has_match = t.type_find_or_create("Has_Match"); + const EdgeType &type_searched_and_clicked = t.type_find_or_create("Searched_And_Clicked"); - auto type_is_employee = t.type_find_or_create("Is_Employee"); - auto type_created = t.type_find_or_create("Created"); + const EdgeType &type_is_employee = t.type_find_or_create("Is_Employee"); + const EdgeType &type_created = t.type_find_or_create("Created"); auto prop_edge_status = t.edge_property_family_get("status") .get(Flags::String) @@ -362,18 +363,21 @@ int main(int argc, char **argv) DbAccessor t(db); int n = 300 * 1000; - vector<pair<Vertex::Accessor, unordered_map<string, double>>> coll; + vector<pair<VertexAccessor, unordered_map<string, double>>> coll; // QUERY BENCHMARK auto begin = clock(); int i = 0; - iter::for_all_fill( + iter::find_fill( t.label_find_or_create("Company").index->for_range_exact(t), [&](auto v) { if (i < n) { coll.push_back(make_pair(v, query(t, v.id()))); + i++; + return false; + } else { + return true; } - i++; }); n = i; clock_t end = clock(); diff --git a/src/api/resources/node.hpp b/src/api/resources/node.hpp index 4e643654c..14362f550 100644 --- a/src/api/resources/node.hpp +++ b/src/api/resources/node.hpp @@ -37,7 +37,7 @@ public: return std::move(vertex_accessor); }, - [&req, &res](Vertex::Accessor&& vertex_accessor) { + [&req, &res](VertexAccessor&& vertex_accessor) { return res.send( http::Status::Created, vertex_create_response(vertex_accessor) @@ -69,7 +69,7 @@ public: return std::move(vertex_accessor); }, - [&req, &res](Vertex::Accessor&& vertex_accessor) { + [&req, &res](VertexAccessor&& vertex_accessor) { if (vertex_accessor.empty()) { return res.send( http::Status::NotFound, @@ -111,7 +111,7 @@ public: return std::move(vertex_accessor); }, - [&req, &res](Vertex::Accessor&& vertex_accessor) { + [&req, &res](VertexAccessor&& vertex_accessor) { if (vertex_accessor.empty()) { return res.send( http::Status::NotFound, diff --git a/src/api/response_json.hpp b/src/api/response_json.hpp index 5c659e271..e54034586 100644 --- a/src/api/response_json.hpp +++ b/src/api/response_json.hpp @@ -29,7 +29,7 @@ using RJStringBuffer = rapidjson::StringBuffer; using RJStringWriter = rapidjson::Writer<RJStringBuffer>; using ptr_RJStringWriter = std::shared_ptr<RJStringWriter>; -std::string vertex_create_response(const Vertex::Accessor& vertex_accessor) +std::string vertex_create_response(const VertexAccessor& vertex_accessor) { // make a string buffer RJStringBuffer buffer; diff --git a/src/database/db.cpp b/src/database/db.cpp index df9abaa74..e00c772ca 100644 --- a/src/database/db.cpp +++ b/src/database/db.cpp @@ -5,3 +5,60 @@ Db::Db() = default; Db::Db(const std::string &name) : name_(name) {} std::string &Db::name() { return name_; } + +template <class TG, class I, class G> +bool Db::create_index_on_vertex_property_family(const char *name, G &coll, + I &create_index) +{ + auto &family = coll.property_family_find_or_create(name); + bool added_index = false; + DbTransaction t(*this, tx_engine.begin([&](auto t) { + added_index = family.index.set_index(create_index(t)); + })); + + if (added_index) { + t.trans.wait_for_active(); + + auto oindex = family.index.get_write(t.trans); + if (oindex.is_present()) { + auto index = oindex.get(); + + for (auto &vr : coll.access()) { + auto v = vr.second.find(t.trans); + if (v != nullptr) { + if (!index->insert(IndexRecord<TG, std::nullptr_t>( + std::nullptr_t(), v, &vr.second))) { + // Index is probably unique. + auto owned_maybe = family.index.remove_index(index); + if (owned_maybe.is_present()) { + garbage.dispose(tx_engine.snapshot(), + owned_maybe.get().release()); + } + + t.trans.abort(); + return false; + } + } + } + + index->activate(); + t.trans.commit(); + return true; + } + } + + t.trans.abort(); + return false; +} + +template <class TG, class K> +bool Db::remove_index(IndexHolder<TG, K> &ih) +{ + auto owned_maybe = ih.remove_index(); + if (owned_maybe.is_present()) { + garbage.dispose(tx_engine.snapshot(), owned_maybe.get().release()); + return true; + } + + return false; +} diff --git a/src/database/db_accessor.cpp b/src/database/db_accessor.cpp index e418a75d5..29bb2bd10 100644 --- a/src/database/db_accessor.cpp +++ b/src/database/db_accessor.cpp @@ -7,35 +7,40 @@ DbAccessor::DbAccessor(Db &db) { } +DbAccessor::DbAccessor(Db &db, tx::Transaction &t) + : db_transaction(DbTransaction(db, t)) +{ +} + // VERTEX METHODS auto DbAccessor::vertex_access() { return iter::make_map( iter::make_iter(this->db_transaction.db.graph.vertices.access()), [&](auto e) -> auto { - return Vertex::Accessor(&(e->second), db_transaction); + return VertexAccessor(&(e->second), db_transaction); }); } -Option<const Vertex::Accessor> DbAccessor::vertex_find(const Id &id) +Option<const VertexAccessor> DbAccessor::vertex_find(const Id &id) { return this->db_transaction.db.graph.vertices.find(db_transaction, id); } -Vertex::Accessor DbAccessor::vertex_insert() +VertexAccessor DbAccessor::vertex_insert() { return this->db_transaction.db.graph.vertices.insert(db_transaction); } // EDGE METHODS -Option<const Edge::Accessor> DbAccessor::edge_find(const Id &id) +Option<const EdgeAccessor> DbAccessor::edge_find(const Id &id) { return db_transaction.db.graph.edges.find(db_transaction, id); } -Edge::Accessor DbAccessor::edge_insert(Vertex::Accessor const &from, - Vertex::Accessor const &to) +EdgeAccessor DbAccessor::edge_insert(VertexAccessor const &from, + VertexAccessor const &to) { auto edge_accessor = db_transaction.db.graph.edges.insert( db_transaction, from.vlist, to.vlist); @@ -67,30 +72,55 @@ bool DbAccessor::type_contains(const char *name) } // PROPERTY METHODS -PropertyFamily &DbAccessor::vertex_property_family_get(const std::string &name) +VertexPropertyFamily & +DbAccessor::vertex_property_family_get(const std::string &name) { return db_transaction.db.graph.vertices.property_family_find_or_create( name); } -PropertyFamily &DbAccessor::edge_property_family_get(const std::string &name) +EdgePropertyFamily & +DbAccessor::edge_property_family_get(const std::string &name) { return db_transaction.db.graph.edges.property_family_find_or_create(name); } // PROPERTY HELPER METHODS -PropertyFamily::PropertyType::PropertyFamilyKey +VertexPropertyFamily::PropertyType::PropertyFamilyKey DbAccessor::vertex_property_key(const std::string &name, Type type) { return vertex_property_family_get(name).get(type).family_key(); } -PropertyFamily::PropertyType::PropertyFamilyKey +EdgePropertyFamily::PropertyType::PropertyFamilyKey DbAccessor::edge_property_key(const std::string &name, Type type) { return edge_property_family_get(name).get(type).family_key(); } +template <class T> +VertexPropertyFamily::PropertyType::PropertyTypeKey<T> +DbAccessor::vertex_property_key(const std::string &name) +{ + return vertex_property_family_get(name).get(T::type).template type_key<T>(); +} + +template <class T> +EdgePropertyFamily::PropertyType::PropertyTypeKey<T> +DbAccessor::edge_property_key(const std::string &name) +{ + return edge_property_family_get(name).get(T::type).template type_key<T>(); +} + // TRANSACTION METHODS -void DbAccessor::commit() { db_transaction.trans.commit(); } +bool DbAccessor::commit() +{ + if (db_transaction.update_indexes()) { + db_transaction.trans.commit(); + return true; + } else { + db_transaction.trans.abort(); + return false; + } +} void DbAccessor::abort() { db_transaction.trans.abort(); } diff --git a/src/database/db_transaction.cpp b/src/database/db_transaction.cpp new file mode 100644 index 000000000..1b2d3fd7d --- /dev/null +++ b/src/database/db_transaction.cpp @@ -0,0 +1,63 @@ +#include "database/db_transaction.hpp" + +#include "storage/edge.hpp" +#include "storage/edge_type/edge_type.hpp" +#include "storage/label/label.hpp" +#include "storage/vertex.hpp" + +#define TRY(x) \ + if (!x) { \ + return false; \ + } + +template <class TG, class IU> +bool update_property_indexes(IU &iu, const tx::Transaction &t) +{ + for (auto kp : iu.record->data.props) { + + // FamilyProperty index + auto opi = kp.first.get_family().index.get_write(t); + if (opi.is_present()) { + TRY(opi.get()->insert(IndexRecord<TG, std::nullptr_t>( + std::nullptr_t(), iu.record, iu.vlist))); + } + + // TODO: other properti indexes + } + + return true; +} + +bool DbTransaction::update_indexes() +{ + while (!index_updates.empty()) { + auto iu = index_updates.back(); + + if (iu.tag == IndexUpdate::EDGE) { + auto e = iu.e; + + // TODO: This could be done in batch + // NOTE: This assumes that type index is created with the database. + TRY(e.record->data.edge_type->index->insert( + EdgeTypeIndexRecord(std::nullptr_t(), e.record, e.vlist))); + + TRY(update_property_indexes<TypeGroupEdge>(e, trans)); + + } else { + auto v = iu.v; + + for (auto l : v.record->data.labels()) { + // TODO: This could be done in batch + // NOTE: This assumes that label index is created with the + // database. + TRY(l.get().index->insert( + LabelIndexRecord(std::nullptr_t(), v.record, v.vlist))); + } + + TRY(update_property_indexes<TypeGroupVertex>(v, trans)); + } + + index_updates.pop_back(); + } + return true; +} diff --git a/src/import/base_import.hpp b/src/import/base_import.hpp index 8b577405e..20acb7904 100644 --- a/src/import/base_import.hpp +++ b/src/import/base_import.hpp @@ -20,7 +20,7 @@ using namespace std; -static Option<Vertex::Accessor> empty_op_vacc; +static Option<VertexAccessor> empty_op_vacc; // Base importer with common facilities. class BaseImporter @@ -186,7 +186,7 @@ public: // } } - Option<Vertex::Accessor> const &get_vertex(size_t id) + Option<VertexAccessor> const &get_vertex(size_t id) { if (vertices.size() > id) { return vertices[id]; @@ -214,5 +214,5 @@ protected: ostream &err_stream; // All created vertices which have import local id - vector<Option<Vertex::Accessor>> vertices; + vector<Option<VertexAccessor>> vertices; }; diff --git a/src/import/csv_import.hpp b/src/import/csv_import.hpp index 865c21b52..731d13163 100644 --- a/src/import/csv_import.hpp +++ b/src/import/csv_import.hpp @@ -50,18 +50,18 @@ public: // Loads data from stream and returns number of loaded vertexes. size_t import_vertices(std::fstream &file) { - return import(file, create_vertex, true); + return import<TypeGroupVertex>(file, create_vertex, true); } // Loads data from stream and returns number of loaded edges. size_t import_edges(std::fstream &file) { - return import(file, create_edge, false); + return import<TypeGroupEdge>(file, create_edge, false); } private: // Loads data from file and returns number of loaded name. - template <class F> + template <class TG, class F> size_t import(std::fstream &file, F f, bool vertex) { string line; @@ -81,7 +81,7 @@ private: } for (auto p : sub_str) { - auto o = get_filler(p, tmp, vertex); + auto o = get_filler<TG>(p, tmp, vertex); if (o.is_present()) { fillers.push_back(o.take()); } else { @@ -132,8 +132,7 @@ private: if (id.is_present()) { if (im->vertices.size() <= id.get()) { - Option<Vertex::Accessor> empty = - make_option<Vertex::Accessor>(); + Option<VertexAccessor> empty = make_option<VertexAccessor>(); im->vertices.insert(im->vertices.end(), id.get() - im->vertices.size() + 1, empty); } @@ -166,7 +165,31 @@ private: } } + // template <typename F> + // Option<unique_ptr<Filler>> make_filler_property(bool vertex, + // const char name, Flags + // type) + // { + // if (vertex) { + // std::unique_ptr<Filler> f( + // F(db.vertex_property_key(name, Type(type)))); + // return make_option(std::move(f)); + // } else { + // std::unique_ptr<Filler> f( + // F(db.edge_property_key(name, Type(type)))); + // return make_option(std::move(f)); + // } + // } + + template <class TG> + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey + prop_key(const char *name, Flags type) + { + assert(false); + } + // Returns filler for name:type in header_part. None if error occured. + template <class TG> Option<unique_ptr<Filler>> get_filler(char *header_part, vector<char *> &tmp_vec, bool vertex) { @@ -193,20 +216,20 @@ private: // cout << name << " # " << type << endl; - auto prop_key = [&](auto name, auto type) -> auto - { - if (vertex) { - return db.vertex_property_key(name, Type(type)); - } else { - return db.edge_property_key(name, Type(type)); - } - }; + // auto prop_key = [&](auto name, auto type) -> auto + // { + // if (vertex) { + // return db.vertex_property_key(name, Type(type)); + // } else { + // return db.edge_property_key(name, Type(type)); + // } + // }; if (equal_str(type, "id")) { std::unique_ptr<Filler> f( - name[0] == '\0' - ? new IdFiller() - : new IdFiller(make_option(prop_key(name, Flags::Int64)))); + name[0] == '\0' ? new IdFiller<TG>() + : new IdFiller<TG>(make_option( + prop_key<TG>(name, Flags::Int64)))); return make_option(std::move(f)); } else if (equal_str(type, "start_id") || equal_str(type, "from_id") || @@ -234,63 +257,69 @@ private: // *********************** PROPERTIES } else if (equal_str(type, "bool")) { + // return make_filler_property<BoolFiller>(vertex, name, + // Flags::Bool); std::unique_ptr<Filler> f( - new BoolFiller(prop_key(name, Flags::Bool))); + new BoolFiller<TG>(prop_key<TG>(name, Flags::Bool))); return make_option(std::move(f)); } else if (equal_str(type, "double")) { std::unique_ptr<Filler> f( - new DoubleFiller(prop_key(name, Flags::Double))); + new DoubleFiller<TG>(prop_key<TG>(name, Flags::Double))); return make_option(std::move(f)); } else if (equal_str(type, "float")) { std::unique_ptr<Filler> f( - new FloatFiller(prop_key(name, Flags::Float))); + new FloatFiller<TG>(prop_key<TG>(name, Flags::Float))); return make_option(std::move(f)); } else if (equal_str(type, "int")) { std::unique_ptr<Filler> f( - new Int32Filler(prop_key(name, Flags::Int32))); + new Int32Filler<TG>(prop_key<TG>(name, Flags::Int32))); return make_option(std::move(f)); } else if (equal_str(type, "long")) { std::unique_ptr<Filler> f( - new Int64Filler(prop_key(name, Flags::Int64))); + new Int64Filler<TG>(prop_key<TG>(name, Flags::Int64))); return make_option(std::move(f)); } else if (equal_str(type, "string")) { std::unique_ptr<Filler> f( - new StringFiller(prop_key(name, Flags::String))); + new StringFiller<TG>(prop_key<TG>(name, Flags::String))); return make_option(std::move(f)); } else if (equal_str(type, "bool[]")) { - std::unique_ptr<Filler> f(make_array_filler<bool, ArrayBool>( - *this, prop_key(name, Flags::ArrayBool), to_bool)); + std::unique_ptr<Filler> f(make_array_filler<TG, bool, ArrayBool>( + *this, prop_key<TG>(name, Flags::ArrayBool), to_bool)); return make_option(std::move(f)); } else if (equal_str(type, "float[]")) { - std::unique_ptr<Filler> f(make_array_filler<float, ArrayFloat>( - *this, prop_key(name, Flags::ArrayFloat), to_float)); + std::unique_ptr<Filler> f(make_array_filler<TG, float, ArrayFloat>( + *this, prop_key<TG>(name, Flags::ArrayFloat), to_float)); return make_option(std::move(f)); } else if (equal_str(type, "double[]")) { - std::unique_ptr<Filler> f(make_array_filler<double, ArrayDouble>( - *this, prop_key(name, Flags::ArrayDouble), to_double)); + std::unique_ptr<Filler> f( + make_array_filler<TG, double, ArrayDouble>( + *this, prop_key<TG>(name, Flags::ArrayDouble), to_double)); return make_option(std::move(f)); } else if (equal_str(type, "int[]")) { - std::unique_ptr<Filler> f(make_array_filler<int32_t, ArrayInt32>( - *this, prop_key(name, Flags::ArrayInt32), to_int32)); + std::unique_ptr<Filler> f( + make_array_filler<TG, int32_t, ArrayInt32>( + *this, prop_key<TG>(name, Flags::ArrayInt32), to_int32)); return make_option(std::move(f)); } else if (equal_str(type, "long[]")) { - std::unique_ptr<Filler> f(make_array_filler<int64_t, ArrayInt64>( - *this, prop_key(name, Flags::ArrayInt64), to_int64)); + std::unique_ptr<Filler> f( + make_array_filler<TG, int64_t, ArrayInt64>( + *this, prop_key<TG>(name, Flags::ArrayInt64), to_int64)); return make_option(std::move(f)); } else if (equal_str(type, "string[]")) { - std::unique_ptr<Filler> f(make_array_filler<string, ArrayString>( - *this, prop_key(name, Flags::ArrayString), to_string)); + std::unique_ptr<Filler> f( + make_array_filler<TG, string, ArrayString>( + *this, prop_key<TG>(name, Flags::ArrayString), to_string)); return make_option(std::move(f)); } else { @@ -315,6 +344,20 @@ private: } }; +template <> +PropertyFamily<TypeGroupVertex>::PropertyType::PropertyFamilyKey +CSVImporter::prop_key<TypeGroupVertex>(const char *name, Flags type) +{ + return db.vertex_property_key(name, Type(type)); +} + +template <> +PropertyFamily<TypeGroupEdge>::PropertyType::PropertyFamilyKey +CSVImporter::prop_key<TypeGroupEdge>(const char *name, Flags type) +{ + return db.edge_property_key(name, Type(type)); +} + // Imports all -v "vertex_file_path.csv" vertices and -e "edge_file_path.csv" // edges from specified files. Also defines arguments -d, -ad, -w, -err, -info. // -d delimiter => sets delimiter for parsing .csv files. Default is , diff --git a/src/import/element_skeleton.hpp b/src/import/element_skeleton.hpp index 9e7b4f3b1..52c005a62 100644 --- a/src/import/element_skeleton.hpp +++ b/src/import/element_skeleton.hpp @@ -14,20 +14,36 @@ class ElementSkeleton { public: - Prop(PropertyFamily::PropertyType::PropertyFamilyKey key, + Prop(VertexPropertyFamily::PropertyType::PropertyFamilyKey key, Option<std::shared_ptr<Property>> &&prop) - : key(key), prop(std::move(prop)) + : key_v(key), prop(std::move(prop)) { } - PropertyFamily::PropertyType::PropertyFamilyKey key; + Prop(EdgePropertyFamily::PropertyType::PropertyFamilyKey key, + Option<std::shared_ptr<Property>> &&prop) + : key_e(key), prop(std::move(prop)) + { + } + + union + { + VertexPropertyFamily::PropertyType::PropertyFamilyKey key_v; + EdgePropertyFamily::PropertyType::PropertyFamilyKey key_e; + }; Option<std::shared_ptr<Property>> prop; }; public: ElementSkeleton(DbAccessor &db) : db(db){}; - void add_property(PropertyFamily::PropertyType::PropertyFamilyKey key, + void add_property(VertexPropertyFamily::PropertyType::PropertyFamilyKey key, + std::shared_ptr<Property> &&prop) + { + properties.push_back(Prop(key, make_option(std::move(prop)))); + } + + void add_property(EdgePropertyFamily::PropertyType::PropertyFamilyKey key, std::shared_ptr<Property> &&prop) { properties.push_back(Prop(key, make_option(std::move(prop)))); @@ -42,17 +58,17 @@ public: void set_type(EdgeType const &type) { this->type = make_option(&type); } - void set_from(Vertex::Accessor &&va) + void set_from(VertexAccessor &&va) { - from_va = make_option<Vertex::Accessor>(std::move(va)); + from_va = make_option<VertexAccessor>(std::move(va)); } - void set_to(Vertex::Accessor &&va) + void set_to(VertexAccessor &&va) { - to_va = make_option<Vertex::Accessor>(std::move(va)); + to_va = make_option<VertexAccessor>(std::move(va)); } - Vertex::Accessor add_vertex() + VertexAccessor add_vertex() { auto va = db.vertex_insert(); @@ -60,7 +76,11 @@ public: // std::cout << *l << std::endl; va.add_label(*l); } - add_propreties(va); + + for (auto prop : properties) { + assert(prop.prop.is_present()); + va.set(prop.key_v, prop.prop.take()); + } return va; } @@ -79,7 +99,11 @@ public: if (type.is_present()) { ve.edge_type(*type.get()); } - add_propreties(ve); + + for (auto prop : properties) { + assert(prop.prop.is_present()); + ve.set(prop.key_e, prop.prop.take()); + } return make_option<std::string>(); } @@ -87,8 +111,8 @@ public: void clear() { el_id = make_option<size_t>(); - to_va = make_option<Vertex::Accessor>(); - from_va = make_option<Vertex::Accessor>(); + to_va = make_option<VertexAccessor>(); + from_va = make_option<VertexAccessor>(); type = make_option<EdgeType const *>(); labels.clear(); properties.clear(); @@ -98,20 +122,20 @@ public: Option<size_t> element_id() { return el_id; } private: - template <class A> - void add_propreties(A &ra) - { - for (auto prop : properties) { - assert(prop.prop.is_present()); - ra.set(prop.key, prop.prop.take()); - } - } + // template <class A> + // void add_propreties(A &ra) + // { + // for (auto prop : properties) { + // assert(prop.prop.is_present()); + // ra.set(prop.key, prop.prop.take()); + // } + // } DbAccessor &db; Option<size_t> el_id; - Option<Vertex::Accessor> to_va; - Option<Vertex::Accessor> from_va; + Option<VertexAccessor> to_va; + Option<VertexAccessor> from_va; Option<EdgeType const *> type; std::vector<Label const *> labels; std::vector<Prop> properties; diff --git a/src/import/fillings/array.hpp b/src/import/fillings/array.hpp index e8ebc2935..dc9327a2b 100644 --- a/src/import/fillings/array.hpp +++ b/src/import/fillings/array.hpp @@ -4,14 +4,15 @@ #include "import/fillings/common.hpp" #include "import/fillings/filler.hpp" -template <class T, class A> +template <class TG, class T, class A> class ArrayFiller : public Filler { public: - ArrayFiller(BaseImporter &db, - PropertyFamily::PropertyType::PropertyFamilyKey key, - T (*f)(const char *)) + ArrayFiller( + BaseImporter &db, + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key, + T (*f)(const char *)) : bim(db), key(key), f(f) { } @@ -36,15 +37,16 @@ public: private: BaseImporter &bim; - PropertyFamily::PropertyType::PropertyFamilyKey key; + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key; vector<char *> sub_str; T (*f)(const char *); }; -template <class T, class A> -auto make_array_filler(BaseImporter &db, - PropertyFamily::PropertyType::PropertyFamilyKey key, - T (*f)(const char *)) +template <class TG, class T, class A> +auto make_array_filler( + BaseImporter &db, + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key, + T (*f)(const char *)) { - return new ArrayFiller<T, A>(db, key, f); + return new ArrayFiller<TG, T, A>(db, key, f); } diff --git a/src/import/fillings/bool.hpp b/src/import/fillings/bool.hpp index ef68ac63f..2fe9032e9 100644 --- a/src/import/fillings/bool.hpp +++ b/src/import/fillings/bool.hpp @@ -6,11 +6,13 @@ #include "storage/model/properties/flags.hpp" #include "storage/model/properties/property_family.hpp" +template <class TG> class BoolFiller : public Filler { public: - BoolFiller(PropertyFamily::PropertyType::PropertyFamilyKey key) : key(key) + BoolFiller(typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key) + : key(key) { } @@ -26,5 +28,5 @@ public: } private: - PropertyFamily::PropertyType::PropertyFamilyKey key; + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key; }; diff --git a/src/import/fillings/double.hpp b/src/import/fillings/double.hpp index 222cdc75f..3eaa1884a 100644 --- a/src/import/fillings/double.hpp +++ b/src/import/fillings/double.hpp @@ -6,11 +6,14 @@ #include "storage/model/properties/flags.hpp" #include "storage/model/properties/property_family.hpp" +template <class TG> class DoubleFiller : public Filler { public: - DoubleFiller(PropertyFamily::PropertyType::PropertyFamilyKey key) : key(key) + DoubleFiller( + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key) + : key(key) { } // Fills skeleton with data from str. Returns error description if @@ -25,5 +28,5 @@ public: } private: - PropertyFamily::PropertyType::PropertyFamilyKey key; + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key; }; diff --git a/src/import/fillings/float.hpp b/src/import/fillings/float.hpp index b5f0c6ca8..e3d6f7511 100644 --- a/src/import/fillings/float.hpp +++ b/src/import/fillings/float.hpp @@ -6,11 +6,14 @@ #include "storage/model/properties/flags.hpp" #include "storage/model/properties/property_family.hpp" +template <class TG> class FloatFiller : public Filler { public: - FloatFiller(PropertyFamily::PropertyType::PropertyFamilyKey key) : key(key) + FloatFiller( + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key) + : key(key) { } // Fills skeleton with data from str. Returns error description if @@ -25,5 +28,5 @@ public: } private: - PropertyFamily::PropertyType::PropertyFamilyKey key; + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key; }; diff --git a/src/import/fillings/from.hpp b/src/import/fillings/from.hpp index 92b9c6b4f..c5569338a 100644 --- a/src/import/fillings/from.hpp +++ b/src/import/fillings/from.hpp @@ -18,9 +18,9 @@ public: { if (str[0] != '\0') { auto id = atol(str); - Option<Vertex::Accessor> const &oav = bim.get_vertex(id); + Option<VertexAccessor> const &oav = bim.get_vertex(id); if (oav.is_present()) { - data.set_from(Vertex::Accessor(oav.get())); + data.set_from(VertexAccessor(oav.get())); return make_option<std::string>(); } else { return make_option( diff --git a/src/import/fillings/id.hpp b/src/import/fillings/id.hpp index 8b90489d4..11df0231a 100644 --- a/src/import/fillings/id.hpp +++ b/src/import/fillings/id.hpp @@ -2,16 +2,20 @@ #include "import/fillings/filler.hpp" +template <class TG> class IdFiller : public Filler { public: IdFiller() - : key(make_option<PropertyFamily::PropertyType::PropertyFamilyKey>()) + : key(make_option< + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey>()) { } - IdFiller(Option<PropertyFamily::PropertyType::PropertyFamilyKey> key) + IdFiller( + Option<typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey> + key) : key(key) { assert(!key.is_present() || @@ -34,5 +38,5 @@ public: } private: - Option<PropertyFamily::PropertyType::PropertyFamilyKey> key; + Option<typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey> key; }; diff --git a/src/import/fillings/int32.hpp b/src/import/fillings/int32.hpp index f06cd2f8d..38d7fa46c 100644 --- a/src/import/fillings/int32.hpp +++ b/src/import/fillings/int32.hpp @@ -6,11 +6,14 @@ #include "storage/model/properties/flags.hpp" #include "storage/model/properties/property_family.hpp" +template <class TG> class Int32Filler : public Filler { public: - Int32Filler(PropertyFamily::PropertyType::PropertyFamilyKey key) : key(key) + Int32Filler( + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key) + : key(key) { } @@ -26,5 +29,5 @@ public: } private: - PropertyFamily::PropertyType::PropertyFamilyKey key; + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key; }; diff --git a/src/import/fillings/int64.hpp b/src/import/fillings/int64.hpp index 7859d84fa..897e32368 100644 --- a/src/import/fillings/int64.hpp +++ b/src/import/fillings/int64.hpp @@ -6,11 +6,14 @@ #include "storage/model/properties/flags.hpp" #include "storage/model/properties/property_family.hpp" +template <class TG> class Int64Filler : public Filler { public: - Int64Filler(PropertyFamily::PropertyType::PropertyFamilyKey key) : key(key) + Int64Filler( + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key) + : key(key) { } // Fills skeleton with data from str. Returns error description if @@ -25,5 +28,5 @@ public: } private: - PropertyFamily::PropertyType::PropertyFamilyKey key; + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key; }; diff --git a/src/import/fillings/string.hpp b/src/import/fillings/string.hpp index a665b25b0..8e60da19f 100644 --- a/src/import/fillings/string.hpp +++ b/src/import/fillings/string.hpp @@ -6,11 +6,14 @@ #include "storage/model/properties/flags.hpp" #include "storage/model/properties/property_family.hpp" +template <class TG> class StringFiller : public Filler { public: - StringFiller(PropertyFamily::PropertyType::PropertyFamilyKey key) : key(key) + StringFiller( + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key) + : key(key) { } // Fills skeleton with data from str. Returns error description if @@ -25,5 +28,5 @@ public: } private: - PropertyFamily::PropertyType::PropertyFamilyKey key; + typename PropertyFamily<TG>::PropertyType::PropertyFamilyKey key; }; diff --git a/src/import/fillings/to.hpp b/src/import/fillings/to.hpp index 45b8ff067..c07db3484 100644 --- a/src/import/fillings/to.hpp +++ b/src/import/fillings/to.hpp @@ -18,9 +18,9 @@ public: { if (str[0] != '\0') { auto id = atol(str); - Option<Vertex::Accessor> const &oav = bim.get_vertex(id); + Option<VertexAccessor> const &oav = bim.get_vertex(id); if (oav.is_present()) { - data.set_to(Vertex::Accessor(oav.get())); + data.set_to(VertexAccessor(oav.get())); return make_option<std::string>(); } else { return make_option( diff --git a/src/query_engine/util.cpp b/src/query_engine/util.cpp index 2efcaf5f3..e7f855fbf 100644 --- a/src/query_engine/util.cpp +++ b/src/query_engine/util.cpp @@ -1,6 +1,10 @@ #include "query_engine/util.hpp" -void print_props(const Properties &properties) +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" + +template <class T> +void print_props(const Properties<T> &properties) { StringBuffer buffer; JsonWriter<StringBuffer> writer(buffer); @@ -8,16 +12,42 @@ void print_props(const Properties &properties) cout << buffer.str() << endl; } -void cout_properties(const Properties &properties) +template <class T> +void cout_properties(const Properties<T> &properties) { ConsoleWriter writer; properties.accept(writer); cout << "----" << endl; } -void cout_property(const prop_key_t &key, const Property &property) +template <class T> +void cout_property( + const typename PropertyFamily<T>::PropertyType::PropertyFamilyKey &key, + const Property &property) { ConsoleWriter writer; - writer.handle(key, property); + writer.handle<T>(key, property); cout << "----" << endl; } + +template void +print_props<TypeGroupEdge>(const Properties<TypeGroupEdge> &properties); + +template void +print_props<TypeGroupVertex>(const Properties<TypeGroupVertex> &properties); + +template void +cout_properties<TypeGroupEdge>(const Properties<TypeGroupEdge> &properties); + +template void +cout_properties<TypeGroupVertex>(const Properties<TypeGroupVertex> &properties); + +template void cout_property<TypeGroupEdge>( + const typename PropertyFamily< + TypeGroupEdge>::PropertyType::PropertyFamilyKey &key, + const Property &property); + +template void cout_property<TypeGroupVertex>( + const typename PropertyFamily< + TypeGroupVertex>::PropertyType::PropertyFamilyKey &key, + const Property &property); diff --git a/src/storage/edge_accessor.cpp b/src/storage/edge_accessor.cpp index 72f7e0582..a095b79fe 100644 --- a/src/storage/edge_accessor.cpp +++ b/src/storage/edge_accessor.cpp @@ -1,22 +1,22 @@ #include "storage/edge_accessor.hpp" -void Edge::Accessor::edge_type(edge_type_ref_t edge_type) +void EdgeAccessor::edge_type(edge_type_ref_t edge_type) { this->record->data.edge_type = &edge_type.get(); } -edge_type_ref_t Edge::Accessor::edge_type() const +edge_type_ref_t EdgeAccessor::edge_type() const { runtime_assert(this->record->data.edge_type != nullptr, "EdgeType is null"); return edge_type_ref_t(*this->record->data.edge_type); } -Vertex::Accessor Edge::Accessor::from() const +VertexAccessor EdgeAccessor::from() const { - return Vertex::Accessor(this->vlist->from(), this->db); + return VertexAccessor(this->vlist->from(), this->db); } -Vertex::Accessor Edge::Accessor::to() const +VertexAccessor EdgeAccessor::to() const { - return Vertex::Accessor(this->vlist->to(), this->db); + return VertexAccessor(this->vlist->to(), this->db); } diff --git a/src/storage/edge_type/edge_type.cpp b/src/storage/edge_type/edge_type.cpp index 6c0fd0996..bacffee3c 100644 --- a/src/storage/edge_type/edge_type.cpp +++ b/src/storage/edge_type/edge_type.cpp @@ -1,9 +1,19 @@ #include "storage/edge_type/edge_type.hpp" -EdgeType::EdgeType() {} -EdgeType::EdgeType(const std::string &id) : id(id) {} -EdgeType::EdgeType(const char *id) : id(std::string(id)) {} -EdgeType::EdgeType(std::string &&id) : id(std::move(id)) {} +EdgeType::EdgeType(const std::string &id) + : id(id), index(std::unique_ptr<type_index_t>(new type_index_t())) +{ +} +EdgeType::EdgeType(const char *id) + : id(std::string(id)), + index(std::unique_ptr<type_index_t>(new type_index_t())) +{ +} +EdgeType::EdgeType(std::string &&id) + : id(std::move(id)), + index(std::unique_ptr<type_index_t>(new type_index_t())) +{ +} bool operator<(const EdgeType &lhs, const EdgeType &rhs) { diff --git a/src/storage/edges.cpp b/src/storage/edges.cpp index b28f1d8b6..e860d8bea 100644 --- a/src/storage/edges.cpp +++ b/src/storage/edges.cpp @@ -1,20 +1,21 @@ #include "storage/edges.hpp" -#include "storage/model/properties/property_family.hpp" + +#include "storage/edge_accessor.hpp" #include "utils/iterator/iterator.hpp" -Option<const Edge::Accessor> Edges::find(DbTransaction &t, const Id &id) +Option<const EdgeAccessor> Edges::find(DbTransaction &t, const Id &id) { auto edges_accessor = edges.access(); auto edges_iterator = edges_accessor.find(id); if (edges_iterator == edges_accessor.end()) - return make_option<const Edge::Accessor>(); + return make_option<const EdgeAccessor>(); - return make_option_const(Edge::Accessor(&edges_iterator->second, t)); + return make_option_const(EdgeAccessor(&edges_iterator->second, t)); } -Edge::Accessor Edges::insert(DbTransaction &t, VertexRecord *from, - VertexRecord *to) +EdgeAccessor Edges::insert(DbTransaction &t, VertexRecord *from, + VertexRecord *to) { // get next vertex id auto next = counter.next(std::memory_order_acquire); @@ -29,16 +30,18 @@ Edge::Accessor Edges::insert(DbTransaction &t, VertexRecord *from, // create new vertex auto inserted_edge_record = result.first; auto edge = inserted_edge_record->second.insert(t.trans); + t.to_update_index<TypeGroupEdge>(&inserted_edge_record->second, edge); - return Edge::Accessor(edge, &inserted_edge_record->second, t); + return EdgeAccessor(edge, &inserted_edge_record->second, t); } -PropertyFamily &Edges::property_family_find_or_create(const std::string &name) +EdgePropertyFamily & +Edges::property_family_find_or_create(const std::string &name) { auto acc = prop_familys.access(); auto it = acc.find(name); if (it == acc.end()) { - PropertyFamily *family = new PropertyFamily(name); + EdgePropertyFamily *family = new EdgePropertyFamily(name); auto res = acc.insert(name, family); if (!res.second) { delete family; diff --git a/src/storage/garbage/garbage.cpp b/src/storage/garbage/garbage.cpp new file mode 100644 index 000000000..4a57bc72c --- /dev/null +++ b/src/storage/garbage/garbage.cpp @@ -0,0 +1,11 @@ +#include "storage/garbage/garbage.hpp" + +void Garbage::dispose(tx::Snapshot<Id> &&snapshot, DeleteSensitive *data) +{ + // TODO: add to list +} + +void Garbage::clean(tx::Engine &engine) +{ + // TODO: iterator throug list and check snapshot +} diff --git a/src/storage/indexes/impl/nonunique_unordered_index.cpp b/src/storage/indexes/impl/nonunique_unordered_index.cpp index 7278d3dc4..9bf7ef37c 100644 --- a/src/storage/indexes/impl/nonunique_unordered_index.cpp +++ b/src/storage/indexes/impl/nonunique_unordered_index.cpp @@ -2,14 +2,28 @@ #include "database/db_accessor.hpp" #include "database/db_transaction.hpp" +#include "storage/edge_accessor.hpp" +#include "storage/edge_record.hpp" +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" +#include "storage/vertex_accessor.hpp" +#include "storage/vertex_record.hpp" #include "utils/iterator/iterator.hpp" +#include "storage/indexes/index_record.cpp" + template <class T, class K> NonUniqueUnorderedIndex<T, K>::NonUniqueUnorderedIndex() : IndexBase<T, K>(false, None) { } +template <class T, class K> +NonUniqueUnorderedIndex<T, K>::NonUniqueUnorderedIndex(tx::Transaction const &t) + : IndexBase<T, K>(false, None, t) +{ +} + template <class T, class K> bool NonUniqueUnorderedIndex<T, K>::insert(IndexRecord<T, K> &&value) { @@ -18,7 +32,7 @@ bool NonUniqueUnorderedIndex<T, K>::insert(IndexRecord<T, K> &&value) } template <class T, class K> -std::unique_ptr<IteratorBase<const typename T::Accessor>> +std::unique_ptr<IteratorBase<const typename T::accessor_t>> NonUniqueUnorderedIndex<T, K>::for_range(DbAccessor &t, Border<K> from, Border<K> to) { @@ -39,14 +53,14 @@ auto NonUniqueUnorderedIndex<T, K>::for_range_exact(DbAccessor &t_v, const IndexRecord<T, K> &r = *it; if (from < r.key && to > r.key && r.is_valid(t.db_transaction.trans)) { - const typename T::Accessor acc = r.access(t.db_transaction); + const typename T::accessor_t acc = r.access(t.db_transaction); it++; return make_option(std::move(acc)); } it++; } - return Option<const typename T::Accessor>(); + return Option<const typename T::accessor_t>(); }); } @@ -56,6 +70,5 @@ void NonUniqueUnorderedIndex<T, K>::clean(DbTransaction &) // TODO: Actual cleaning } -#include "storage/vertex.hpp" -// #include "utils/singleton.hpp" -template class NonUniqueUnorderedIndex<Vertex, std::nullptr_t>; +template class NonUniqueUnorderedIndex<TypeGroupEdge, std::nullptr_t>; +template class NonUniqueUnorderedIndex<TypeGroupVertex, std::nullptr_t>; diff --git a/src/storage/indexes/impl/unique_ordered_index.cpp b/src/storage/indexes/impl/unique_ordered_index.cpp new file mode 100644 index 000000000..62ee11f59 --- /dev/null +++ b/src/storage/indexes/impl/unique_ordered_index.cpp @@ -0,0 +1,91 @@ +#include "storage/indexes/impl/unique_ordered_index.hpp" + +#include "database/db_accessor.hpp" +#include "database/db_transaction.hpp" +#include "storage/edge_accessor.hpp" +#include "storage/edge_record.hpp" +#include "storage/edge_type/edge_type.hpp" +#include "storage/label/label.hpp" +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" +#include "storage/vertex_accessor.hpp" +#include "storage/vertex_record.hpp" +#include "utils/iterator/iterator.hpp" + +#include "storage/indexes/index_record.cpp" + +template <class T, class K> +UniqueOrderedIndex<T, K>::UniqueOrderedIndex(Order order) + : IndexBase<T, K>(true, order) +{ +} + +template <class T, class K> +UniqueOrderedIndex<T, K>::UniqueOrderedIndex(Order order, + tx::Transaction const &t) + : IndexBase<T, K>(true, order, t) +{ +} + +template <class T, class K> +bool UniqueOrderedIndex<T, K>::insert(IndexRecord<T, K> &&value) +{ + if (this->order == Descending) { + value.set_descending(); + } + return set.access().insert(std::move(value)).second; +} + +template <class T, class K> +std::unique_ptr<IteratorBase<const typename T::accessor_t>> +UniqueOrderedIndex<T, K>::for_range(DbAccessor &t, Border<K> from, Border<K> to) +{ + return std::make_unique<decltype( + for_range_exact(t, std::move(from), std::move(to)))>( + for_range_exact(t, std::move(from), std::move(to))); +} + +template <class T, class K> +auto UniqueOrderedIndex<T, K>::for_range_exact(DbAccessor &t_v, + Border<K> from_v, Border<K> to_v) +{ + auto acc = set.access(); + auto begin = acc.cbegin(); + auto end = to_v; + + // Sorted order must be checked + if (this->order == Ascending && from_v.key.is_present()) { + begin = acc.cfind_or_larger(from_v); + } else if (this->order == Descending && to_v.key.is_present()) { + begin = acc.cfind_or_larger(to_v); + end = from_v; + } else { + assert(this->order != None); + } + + return iter::make_iterator([ + it = std::move(begin), b_end = std::move(end), t = t_v, + hold_acc = std::move(acc) + ]() mutable->auto { + while (b_end >= it->key) { + const IndexRecord<T, K> &r = *it; + if (r.is_valid(t.db_transaction.trans)) { + const typename T::accessor_t acc = r.access(t.db_transaction); + it++; + return make_option(std::move(acc)); + } + it++; + } + + return Option<const typename T::accessor_t>(); + }); +} + +template <class T, class K> +void UniqueOrderedIndex<T, K>::clean(DbTransaction &) +{ + // TODO: Actual cleaning +} + +template class UniqueOrderedIndex<TypeGroupEdge, std::nullptr_t>; +template class UniqueOrderedIndex<TypeGroupVertex, std::nullptr_t>; diff --git a/src/storage/indexes/index_base.cpp b/src/storage/indexes/index_base.cpp new file mode 100644 index 000000000..b2712a717 --- /dev/null +++ b/src/storage/indexes/index_base.cpp @@ -0,0 +1,39 @@ +#include "storage/indexes/index_base.hpp" + +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" +#include "transactions/transaction.hpp" + +template <class TG, class K> +IndexBase<TG, K>::IndexBase(bool unique, Order order) + : unique(unique), order(order), created(Id(0)), active(true) +{ +} + +template <class TG, class K> +IndexBase<TG, K>::IndexBase(bool unique, Order order, const tx::Transaction &t) + : unique(unique), order(order), created(t.id) +{ +} + +template <class TG, class K> +void IndexBase<TG, K>::activate() +{ + assert(!can_read()); + active.store(true); +} + +template <class TG, class K> +bool IndexBase<TG, K>::can_read() +{ + return active.load(std::memory_order_acquire); +} + +template <class TG, class K> +bool IndexBase<TG, K>::is_obliged_to_insert(const tx::Transaction &t) +{ + return t.id >= created; +} + +template class IndexBase<TypeGroupEdge, std::nullptr_t>; +template class IndexBase<TypeGroupVertex, std::nullptr_t>; diff --git a/src/storage/indexes/index_holder.cpp b/src/storage/indexes/index_holder.cpp new file mode 100644 index 000000000..de3e0510e --- /dev/null +++ b/src/storage/indexes/index_holder.cpp @@ -0,0 +1,66 @@ +#include "storage/indexes/index_holder.hpp" + +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" + +template <class TG, class K> +IndexBase<TG, K> *npr = (IndexBase<TG, K> *)nullptr; + +template <class TG, class K> +bool IndexHolder<TG, K>::set_index(std::unique_ptr<IndexBase<TG, K>> inx) +{ + if (index.compare_exchange_strong(npr<TG, K>, inx.get())) { + inx.release(); + return true; + } else { + return false; + } +} + +template <class TG, class K> +OptionPtr<IndexBase<TG, K>> IndexHolder<TG, K>::get_read() const +{ + auto loaded = index.load(std::memory_order_acquire); + if (loaded == nullptr || !loaded->can_read()) { + return OptionPtr<IndexBase<TG, K>>(); + } else { + return make_option_ptr(loaded); + } +} + +template <class TG, class K> +OptionPtr<IndexBase<TG, K>> +IndexHolder<TG, K>::get_write(const tx::Transaction &t) const +{ + auto loaded = index.load(std::memory_order_acquire); + if (loaded == nullptr || !loaded->is_obliged_to_insert(t)) { + return OptionPtr<IndexBase<TG, K>>(); + } else { + return make_option_ptr(loaded); + } +} + +template <class TG, class K> +Option<std::unique_ptr<IndexBase<TG, K>>> +IndexHolder<TG, K>::remove_index(IndexBase<TG, K> *expected) +{ + if (index.compare_exchange_strong(expected, nullptr)) { + return make_option(std::unique_ptr<IndexBase<TG, K>>(expected)); + } else { + return make_option(std::unique_ptr<IndexBase<TG, K>>()); + } +} + +template <class TG, class K> +Option<std::unique_ptr<IndexBase<TG, K>>> IndexHolder<TG, K>::remove_index() +{ + auto removed = index.exchange(nullptr); + if (removed == nullptr) { + return make_option<std::unique_ptr<IndexBase<TG, K>>>(); + } else { + return make_option(std::unique_ptr<IndexBase<TG, K>>(removed)); + } +} + +template class IndexHolder<TypeGroupEdge, std::nullptr_t>; +template class IndexHolder<TypeGroupVertex, std::nullptr_t>; diff --git a/src/storage/indexes/index_record.cpp b/src/storage/indexes/index_record.cpp new file mode 100644 index 000000000..c3e42142c --- /dev/null +++ b/src/storage/indexes/index_record.cpp @@ -0,0 +1,46 @@ +#include "storage/indexes/index_record.hpp" + +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" + +template <class TG, class K> +IndexRecord<TG, K>::IndexRecord(K key, typename TG::record_t *record, + typename TG::vlist_t *vlist) + : key(std::move(key)), record(record), vlist(vlist) +{ + assert(record != nullptr); + assert(vlist != nullptr); +} + +template <class TG, class K> +void IndexRecord<TG, K>::set_descending() +{ + descending = true; +} + +template <class TG, class K> +bool IndexRecord<TG, K>::empty() const +{ + return record == nullptr; +} + +template <class TG, class K> +bool IndexRecord<TG, K>::is_valid(tx::Transaction &t) const +{ + assert(!empty()); + return record == vlist->find(t); +} + +template <class TG, class K> +const auto IndexRecord<TG, K>::access(DbTransaction &db) const +{ + return typename TG::accessor_t(record, vlist, db); +} + +#include "storage/edge_accessor.hpp" +#include "storage/edge_record.hpp" +#include "storage/vertex_accessor.hpp" +#include "storage/vertex_record.hpp" + +template class IndexRecord<TypeGroupVertex, std::nullptr_t>; +template class IndexRecord<TypeGroupEdge, std::nullptr_t>; diff --git a/src/storage/indexes/index_update.cpp b/src/storage/indexes/index_update.cpp new file mode 100644 index 000000000..ca7fe1028 --- /dev/null +++ b/src/storage/indexes/index_update.cpp @@ -0,0 +1,14 @@ +#include "storage/indexes/index_update.hpp" + +template <> +IndexUpdate make_index_update(EdgeRecord *vlist, Edge *record) +{ + return IndexUpdate{IndexUpdate::EDGE, .e = IndexUpdateEdge{vlist, record}}; +} + +template <> +IndexUpdate make_index_update(VertexRecord *vlist, Vertex *record) +{ + return IndexUpdate{IndexUpdate::VERTEX, + .v = IndexUpdateVertex{vlist, record}}; +} diff --git a/src/storage/model/properties/properties.cpp b/src/storage/model/properties/properties.cpp index 76c1d2408..61b8a9ed5 100644 --- a/src/storage/model/properties/properties.cpp +++ b/src/storage/model/properties/properties.cpp @@ -4,7 +4,11 @@ #include "storage/model/properties/property_family.hpp" #include "utils/option.hpp" -const Property &Properties::at(PropertyFamily &fam) const +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" + +template <class TG> +const Property &Properties<TG>::at(PropertyFamily<TG> &fam) const { // It doesn't matter whit which type // find is called, thats why getNull @@ -18,7 +22,8 @@ const Property &Properties::at(PropertyFamily &fam) const return *it->second.get(); } -const Property &Properties::at(prop_key_t &key) const +template <class TG> +const Property &Properties<TG>::at(prop_key_t &key) const { auto it = props.find(key); @@ -28,8 +33,9 @@ const Property &Properties::at(prop_key_t &key) const return *it->second.get(); } +template <class TG> template <class T> -auto Properties::at(type_key_t<T> &key) const +auto Properties<TG>::at(type_key_t<T> &key) const { auto f_key = key.family_key(); auto it = props.find(f_key); @@ -41,8 +47,9 @@ auto Properties::at(type_key_t<T> &key) const return make_option(&(it->second.get()->template as<T>().value_ref())); } +template <class TG> template <class T, class... Args> -void Properties::set(type_key_t<T> &key, Args &&... args) +void Properties<TG>::set(type_key_t<T> &key, Args &&... args) { auto value = std::make_shared<T>(std::forward<Args>(args)...); @@ -59,7 +66,8 @@ void Properties::set(type_key_t<T> &key, Args &&... args) } } -void Properties::set(prop_key_t &key, Property::sptr value) +template <class TG> +void Properties<TG>::set(prop_key_t &key, Property::sptr value) { // TODO: There is uneccesary copying of value here. auto result = props.insert(make_pair(key, value)); @@ -72,9 +80,14 @@ void Properties::set(prop_key_t &key, Property::sptr value) } } -void Properties::clear(prop_key_t &key) { props.erase(key); } +template <class TG> +void Properties<TG>::clear(prop_key_t &key) +{ + props.erase(key); +} -void Properties::clear(PropertyFamily &fam) +template <class TG> +void Properties<TG>::clear(PropertyFamily<TG> &fam) { // It doesn't matter whit which type // find is called, thats why getNull @@ -84,9 +97,6 @@ void Properties::clear(PropertyFamily &fam) props.erase(key); } -template <> -inline void Properties::set<Null>(type_key_t<Null> &key) -{ - auto fk = key.family_key(); - clear(fk); -} +template class Properties<TypeGroupEdge>; + +template class Properties<TypeGroupVertex>; diff --git a/src/storage/model/properties/property_family.cpp b/src/storage/model/properties/property_family.cpp index 58c9d8869..fe16a4a59 100644 --- a/src/storage/model/properties/property_family.cpp +++ b/src/storage/model/properties/property_family.cpp @@ -1,19 +1,31 @@ #include "storage/model/properties/property_family.hpp" -PropertyFamily::PropertyFamily(std::string const &name_v) +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" + +template <class T> +PropertyFamily<T>::PropertyFamily(std::string const &name_v) : name_v(std::forward<const std::string>(name_v)) { null_type = &get(Flags::Null); } -PropertyFamily::PropertyFamily(std::string &&name_v) : name_v(std::move(name_v)) + +template <class T> +PropertyFamily<T>::PropertyFamily(std::string &&name_v) + : name_v(std::move(name_v)) { null_type = &get(Flags::Null); } -std::string const &PropertyFamily::name() const { return name_v; } +template <class T> +std::string const &PropertyFamily<T>::name() const +{ + return name_v; +} // Returns type if it exists otherwise creates it. -PropertyFamily::PropertyType &PropertyFamily::get(Type type) +template <class T> +typename PropertyFamily<T>::PropertyType &PropertyFamily<T>::get(Type type) { auto acc = types.access(); auto it = acc.find(type); @@ -26,16 +38,25 @@ PropertyFamily::PropertyType &PropertyFamily::get(Type type) return *(it->second); } -PropertyFamily::PropertyType::PropertyType(PropertyFamily &family, Type type) +template <class T> +PropertyFamily<T>::PropertyType::PropertyType(PropertyFamily &family, Type type) : family(family), type(std::move(type)) { } -bool PropertyFamily::PropertyType::is(Type &t) const { return type == t; } +template <class T> +bool PropertyFamily<T>::PropertyType::is(Type &t) const +{ + return type == t; +} // Returns key ordered on POINTERS to PropertyFamily -PropertyFamily::PropertyType::PropertyFamilyKey -PropertyFamily::PropertyType::family_key() +template <class T> +typename PropertyFamily<T>::PropertyType::PropertyFamilyKey +PropertyFamily<T>::PropertyType::family_key() { return PropertyFamilyKey(*this); } + +template class PropertyFamily<TypeGroupEdge>; +template class PropertyFamily<TypeGroupVertex>; diff --git a/src/storage/record_accessor.cpp b/src/storage/record_accessor.cpp index 21198de4c..422b1b4db 100644 --- a/src/storage/record_accessor.cpp +++ b/src/storage/record_accessor.cpp @@ -1,8 +1,8 @@ #include "storage/record_accessor.hpp" -template <class T, class Derived, class vlist_t> -template <class V> -auto RecordAccessor<T, Derived, vlist_t>::at(type_key_t<V> &key) const -{ - return properties().template at<V>(key); -} +// template <class T, class Derived> +// template <class V> +// auto RecordAccessor<T, Derived>::at(type_key_t<T, V> &key) const +// { +// return properties().template at<V>(key); +// } diff --git a/src/storage/vertex_accessor.cpp b/src/storage/vertex_accessor.cpp index e171aea10..0aae09eb7 100644 --- a/src/storage/vertex_accessor.cpp +++ b/src/storage/vertex_accessor.cpp @@ -1,64 +1,61 @@ -#include "database/db.hpp" #include "storage/vertex_accessor.hpp" + +#include "database/db.hpp" +#include "storage/vertex_record.hpp" #include "storage/vertices.hpp" #include "utils/iterator/iterator.hpp" -size_t Vertex::Accessor::out_degree() const +size_t VertexAccessor::out_degree() const { return this->record->data.out.degree(); } -size_t Vertex::Accessor::in_degree() const +size_t VertexAccessor::in_degree() const { return this->record->data.in.degree(); } -size_t Vertex::Accessor::degree() const { return in_degree() + out_degree(); } +size_t VertexAccessor::degree() const { return in_degree() + out_degree(); } -bool Vertex::Accessor::add_label(const Label &label) +bool VertexAccessor::add_label(const Label &label) { // update vertex - if (this->record->data.labels.add(label)) { - label.index->insert(create_index_record()); - return true; - } - return false; + return this->record->data.labels.add(label); } -bool Vertex::Accessor::remove_label(const Label &label) +bool VertexAccessor::remove_label(const Label &label) { // update vertex return this->record->data.labels.remove(label); } -bool Vertex::Accessor::has_label(const Label &label) const +bool VertexAccessor::has_label(const Label &label) const { return this->record->data.labels.has(label); } -const std::set<label_ref_t> &Vertex::Accessor::labels() const +const std::set<label_ref_t> &VertexAccessor::labels() const { return this->record->data.labels(); } // Returns unfilled accessors -auto Vertex::Accessor::out() const +auto VertexAccessor::out() const { DbTransaction &t = this->db; - return iter::make_map( - iter::make_iter_ref(record->data.out), - [&](auto e) -> auto { return Edge::Accessor(*e, t); }); + return iter::make_map(iter::make_iter_ref(record->data.out), + [&](auto e) -> auto { return EdgeAccessor(*e, t); }); } // Returns unfilled accessors -auto Vertex::Accessor::in() const +auto VertexAccessor::in() const { DbTransaction &t = this->db; return iter::make_map(iter::make_iter_ref(record->data.in), - [&](auto e) -> auto { return Edge::Accessor(e, t); }); + [&](auto e) -> auto { return EdgeAccessor(e, t); }); } -bool Vertex::Accessor::in_contains(Vertex::Accessor const &other) const +bool VertexAccessor::in_contains(VertexAccessor const &other) const { return record->data.in.contains(other.vlist); } diff --git a/src/storage/vertices.cpp b/src/storage/vertices.cpp index 5b6e46040..31be412cc 100644 --- a/src/storage/vertices.cpp +++ b/src/storage/vertices.cpp @@ -1,20 +1,22 @@ #include "storage/vertices.hpp" + +#include "storage/vertex_accessor.hpp" #include "utils/iterator/iterator.hpp" Vertices::vertices_t::Accessor Vertices::access() { return vertices.access(); } -Option<const Vertex::Accessor> Vertices::find(DbTransaction &t, const Id &id) +Option<const VertexAccessor> Vertices::find(DbTransaction &t, const Id &id) { auto vertices_accessor = vertices.access(); auto vertices_iterator = vertices_accessor.find(id); if (vertices_iterator == vertices_accessor.end()) - return make_option<const Vertex::Accessor>(); + return make_option<const VertexAccessor>(); - return make_option_const(Vertex::Accessor(&vertices_iterator->second, t)); + return make_option_const(VertexAccessor(&vertices_iterator->second, t)); } -Vertex::Accessor Vertices::insert(DbTransaction &t) +VertexAccessor Vertices::insert(DbTransaction &t) { // get next vertex id auto next = counter.next(); @@ -30,17 +32,19 @@ Vertex::Accessor Vertices::insert(DbTransaction &t) // create new vertex auto inserted_vertex_record = result.first; auto vertex = inserted_vertex_record->second.insert(t.trans); + t.to_update_index<TypeGroupVertex>(&inserted_vertex_record->second, vertex); - return Vertex::Accessor(vertex, &inserted_vertex_record->second, t); + return VertexAccessor(vertex, &inserted_vertex_record->second, t); } -PropertyFamily & +VertexPropertyFamily & Vertices::property_family_find_or_create(const std::string &name) { auto acc = prop_familys.access(); auto it = acc.find(name); if (it == acc.end()) { - auto family = std::unique_ptr<PropertyFamily>(new PropertyFamily(name)); + auto family = std::unique_ptr<VertexPropertyFamily>( + new VertexPropertyFamily(name)); auto res = acc.insert(name, std::move(family)); it = res.first; } diff --git a/src/transactions/transaction.cpp b/src/transactions/transaction.cpp index 402c0c519..aab629276 100644 --- a/src/transactions/transaction.cpp +++ b/src/transactions/transaction.cpp @@ -1,5 +1,9 @@ #include "transactions/transaction.hpp" +#include <chrono> // std::chrono::seconds + +#include <thread> // std::this_thread::sleep_for + #include "transactions/engine.hpp" namespace tx @@ -11,10 +15,25 @@ Transaction::Transaction(const Id &id, const Snapshot<Id> &snapshot, { } +void Transaction::wait_for_active() +{ + while (snapshot.size() > 0) { + auto id = snapshot.back(); + while (engine.clog.fetch_info(id).is_active()) { + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + snapshot.remove(id); + } +} + +bool Transaction::is_active(const Id &id) const +{ + return snapshot.is_active(id); +} + void Transaction::take_lock(RecordLock &lock) { locks.take(&lock, id); } void Transaction::commit() { engine.commit(*this); } void Transaction::abort() { engine.abort(*this); } - }