From 81abee0d3358571c6927f073543e9f9833a96939 Mon Sep 17 00:00:00 2001
From: Kruno Tomola Fabro <krunotf@memgraph.io>
Date: Tue, 13 Sep 2016 10:59:40 +0100
Subject: [PATCH 1/5] Tmp commit
---
tests/poc/profile.cpp | 0
1 file changed, 0 insertions(+), 0 deletions(-)
create mode 100644 tests/poc/profile.cpp
diff --git a/tests/poc/profile.cpp b/tests/poc/profile.cpp
new file mode 100644
index 000000000..e69de29bb
From e22074ad9f898af77af63db3a51a8170e5211ee8 Mon Sep 17 00:00:00 2001
From: Kruno Tomola Fabro <krunotf@memgraph.io>
Date: Tue, 13 Sep 2016 17:02:04 +0100
Subject: [PATCH 2/5] Added test for snapshot with stored index.
---
include/snapshot/snapshot_engine.hpp | 10 +-
src/snapshot/snapshot_engine.cpp | 50 ++++---
src/storage/indexes/indexes.cpp | 2 +-
tests/integration/snapshot.cpp | 187 +++++++++++++++++++--------
4 files changed, 171 insertions(+), 78 deletions(-)
diff --git a/include/snapshot/snapshot_engine.hpp b/include/snapshot/snapshot_engine.hpp
index 96069bdf2..20d25e475 100644
--- a/include/snapshot/snapshot_engine.hpp
+++ b/include/snapshot/snapshot_engine.hpp
@@ -4,6 +4,7 @@
#include <unordered_map>
#include "logging/default.hpp"
+#include "storage/indexes/index_definition.hpp"
#include "transactions/transaction.hpp"
class SnapshotEncoder;
@@ -42,9 +43,14 @@ private:
void snapshot(DbTransaction const &dt, SnapshotEncoder &snap,
tx::TransactionRead const &old_trans);
- // Loads snapshot. True if success
- bool snapshot_load(DbAccessor &t, SnapshotDecoder &snap);
+ // Loads snapshot. True if success. Returns indexes which were in snapshot.
+ std::vector<IndexDefinition> snapshot_load(DbAccessor &t,
+ SnapshotDecoder &snap);
+ // Adds indexes. Should be called outside transactions.
+ void add_indexes(std::vector<IndexDefinition> &v);
+
+ // Will return different name on every call.
std::string snapshot_file(std::time_t const &now, const char *type);
std::string snapshot_commit_file();
diff --git a/src/snapshot/snapshot_engine.cpp b/src/snapshot/snapshot_engine.cpp
index 397144aaf..8f5fe0449 100644
--- a/src/snapshot/snapshot_engine.cpp
+++ b/src/snapshot/snapshot_engine.cpp
@@ -156,22 +156,18 @@ bool SnapshotEngine::import()
std::fstream::binary);
SnapshotDecoder decoder(snapshot_file);
- if (snapshot_load(t, decoder)) {
- if (t.commit()) {
- logger.info("Succesfully imported snapshot \"{}\"",
- snapshots.back());
- success = true;
- break;
+ auto indexes = snapshot_load(t, decoder);
+ if (t.commit()) {
+ logger.info("Succesfully imported snapshot \"{}\"",
+ snapshots.back());
+ add_indexes(indexes);
+ success = true;
+ break;
- } else {
- logger.info("Unuccesfully tryed to import snapshot "
- "\"{}\" because indexes where unuccesfully "
- "with updating",
- snapshots.back());
- }
} else {
- t.abort();
- logger.info("Unuccesfully tryed to import snapshot \"{}\"",
+ logger.info("Unuccesfully tryed to import snapshot "
+ "\"{}\" because indexes where unuccesfully "
+ "with updating",
snapshots.back());
}
@@ -241,7 +237,8 @@ void SnapshotEngine::snapshot(DbTransaction const &dt, SnapshotEncoder &snap,
snap.end();
}
-bool SnapshotEngine::snapshot_load(DbAccessor &t, SnapshotDecoder &snap)
+std::vector<IndexDefinition>
+SnapshotEngine::snapshot_load(DbAccessor &t, SnapshotDecoder &snap)
{
std::unordered_map<uint64_t, VertexAccessor> vertices;
@@ -268,23 +265,36 @@ bool SnapshotEngine::snapshot_load(DbAccessor &t, SnapshotDecoder &snap)
// Load indexes
snap.start_indexes();
+ std::vector<IndexDefinition> indexes;
while (!snap.end()) {
- // This will add index.
+ indexes.push_back(snap.load_index());
+ }
+
+ return indexes;
+}
+
+void SnapshotEngine::add_indexes(std::vector<IndexDefinition> &v)
+{
+ logger.info("Adding: {} indexes", v.size());
+ for (auto id : v) {
// TODO: It is alright for now to ignore if add_index return false. I am
// not even sure if false should stop snapshot loading.
- if (!db.indexes().add_index(snap.load_index())) {
+ if (!db.indexes().add_index(id)) {
logger.warn("Failed to add index, but still continuing with "
"loading snapshot");
}
}
-
- return true;
}
std::string SnapshotEngine::snapshot_file(std::time_t const &now,
const char *type)
{
- return snapshot_db_dir() + "/" + std::to_string(now) + "_" + type;
+ auto now_nano = std::chrono::time_point_cast<std::chrono::nanoseconds>(
+ std::chrono::high_resolution_clock::now())
+ .time_since_epoch()
+ .count();
+ return snapshot_db_dir() + "/" + std::to_string(now) + "_" +
+ std::to_string(now_nano) + "_" + type;
}
std::string SnapshotEngine::snapshot_commit_file()
diff --git a/src/storage/indexes/indexes.cpp b/src/storage/indexes/indexes.cpp
index bdad611d8..72d17abd0 100644
--- a/src/storage/indexes/indexes.cpp
+++ b/src/storage/indexes/indexes.cpp
@@ -13,7 +13,7 @@ bool Indexes::add_index(IndexDefinition id)
// Creates transaction and during it's creation adds index into it's
// place. Also created finish closure which will add necessary elements
// into index.
- DbTransaction t(db, db.tx_engine.begin([&, id](auto &t) mutable {
+ DbTransaction t(db, db.tx_engine.begin([&](auto &t) mutable {
size_t code = id.loc.location_code();
switch (code) {
diff --git a/tests/integration/snapshot.cpp b/tests/integration/snapshot.cpp
index 614068585..c68c1a5bd 100644
--- a/tests/integration/snapshot.cpp
+++ b/tests/integration/snapshot.cpp
@@ -7,6 +7,7 @@
#include "logging/default.hpp"
#include "logging/streams/stdout.hpp"
#include "query_engine/query_stripper.hpp"
+#include "storage/indexes/indexes.hpp"
#include "utils/sysinfo/memory.hpp"
// Returns uniform random size_t generator from range [0,n>
@@ -17,9 +18,10 @@ auto rand_gen(size_t n)
return std::bind(distribution, generator);
}
-template <class S, class Q>
-void run(size_t n, std::string &query, S &stripper, Q &qf)
+void run(size_t n, std::string &query, Db &db)
{
+ auto stripper = make_query_stripper(TK_LONG, TK_FLOAT, TK_STR, TK_BOOL);
+ auto qf = load_queries(barrier::trans(db));
auto stripped = stripper.strip(query);
std::cout << "Running query [" << stripped.hash << "] for " << n << " time."
<< std::endl;
@@ -29,10 +31,10 @@ void run(size_t n, std::string &query, S &stripper, Q &qf)
}
}
-template <class S, class Q>
-void add_edge(size_t n, Db &db, S &stripper, Q &qf)
+void add_edge(size_t n, Db &db)
{
-
+ auto stripper = make_query_stripper(TK_LONG, TK_FLOAT, TK_STR, TK_BOOL);
+ auto qf = load_queries(barrier::trans(db));
std::string query = "MATCH (n1), (n2) WHERE ID(n1)=0 AND "
"ID(n2)=1 CREATE (n1)<-[r:IS {age: "
"25,weight: 70}]-(n2) RETURN r";
@@ -55,6 +57,45 @@ void add_edge(size_t n, Db &db, S &stripper, Q &qf)
}
}
+void add_property(Db &db, StoredProperty<TypeGroupVertex> &prop)
+{
+ DbAccessor t(db);
+
+ t.vertex_access().fill().for_all([&](auto va) { va.set(prop); });
+
+ assert(t.commit());
+}
+
+void add_property_different_int(Db &db, PropertyFamily<TypeGroupVertex> &f)
+{
+ DbAccessor t(db);
+
+ auto key = f.get(Int64::type).family_key();
+
+ size_t i = 0;
+ t.vertex_access().fill().for_all([&](auto va) mutable {
+ va.set(StoredProperty<TypeGroupVertex>(Int64(i), key));
+ i++;
+ });
+
+ assert(t.commit());
+}
+
+size_t size(Db &db, IndexHolder<TypeGroupVertex, std::nullptr_t> &h)
+{
+ DbAccessor t(db);
+
+ size_t count = 0;
+ auto oin = h.get_read();
+ if (oin.is_present()) {
+ oin.get()->for_range(t).for_all([&](auto va) mutable { count++; });
+ }
+
+ t.commit();
+
+ return count;
+}
+
void assert_empty(Db &db)
{
assert(db.graph.vertices.access().size() == 0);
@@ -75,14 +116,13 @@ void clean_edge(Db &db)
t.trans.commit();
}
-template <class S, class Q>
-void clear_database(Db &db, S &stripper, Q &qf)
+void clear_database(Db &db)
{
std::string delete_all_vertices = "MATCH (n) DELETE n";
std::string delete_all_edges = "MATCH ()-[r]-() DELETE r";
- run(1, delete_all_edges, stripper, qf);
- run(1, delete_all_vertices, stripper, qf);
+ run(1, delete_all_edges, db);
+ run(1, delete_all_vertices, db);
clean_vertex(db);
clean_edge(db);
assert_empty(db);
@@ -132,12 +172,6 @@ int main(void)
size_t cvl_n = 1000;
- Db db("snapshot");
-
- auto query_functions = load_queries(barrier::trans(db));
-
- auto stripper = make_query_stripper(TK_LONG, TK_FLOAT, TK_STR, TK_BOOL);
-
std::string create_vertex_label =
"CREATE (n:LABEL {name: \"cleaner_test\"}) RETURN n";
std::string create_vertex_other =
@@ -145,51 +179,94 @@ int main(void)
std::string delete_label_vertices = "MATCH (n:LABEL) DELETE n";
std::string delete_all_vertices = "MATCH (n) DELETE n";
- // ********************* MAKE SURE THAT DB IS EMPTY **********************//
- clear_database(db, stripper, query_functions);
-
- std::cout << "TEST1" << std::endl;
// ******************************* TEST 1 ********************************//
- // make snapshot of empty db
- // add vertexs
- // add edges
- // empty database
- // import snapshot
- // assert database empty
-
- db.snap_engine.make_snapshot();
- run(cvl_n, create_vertex_label, stripper, query_functions);
- add_edge(cvl_n, db, stripper, query_functions);
- clear_database(db, stripper, query_functions);
- db.snap_engine.import();
- assert_empty(db);
-
- std::cout << "TEST2" << std::endl;
- // ******************************* TEST 2 ********************************//
- // add vertexs
- // add edges
- // make snapshot of db
- // empty database
- // import snapshot
- // create new db
- // compare database with new db
-
- run(cvl_n, create_vertex_label, stripper, query_functions);
- add_edge(cvl_n, db, stripper, query_functions);
- db.snap_engine.make_snapshot();
- clear_database(db, stripper, query_functions);
- db.snap_engine.import();
{
- Db db2("snapshot");
- assert(equal(db, db2));
+ std::cout << "TEST1" << std::endl;
+ // make snapshot of empty db
+ // add vertexs
+ // add edges
+ // empty database
+ // import snapshot
+ // assert database empty
+
+ Db db("snapshot", false);
+ db.snap_engine.make_snapshot();
+ run(cvl_n, create_vertex_label, db);
+ add_edge(cvl_n, db);
+ clear_database(db);
+ db.snap_engine.import();
+ assert_empty(db);
}
- std::cout << "TEST3" << std::endl;
- // ******************************* TEST 3 ********************************//
- // compare database with different named database
+ // ******************************* TEST 2 ********************************//
{
- Db db2("not_snapshot");
- assert(!equal(db, db2));
+ std::cout << "TEST2" << std::endl;
+ // add vertexs
+ // add edges
+ // make snapshot of db
+ // empty database
+ // import snapshot
+ // create new db
+ // compare database with new db
+ Db db("snapshot", false);
+ run(cvl_n, create_vertex_label, db);
+ add_edge(cvl_n, db);
+ db.snap_engine.make_snapshot();
+ clear_database(db);
+ db.snap_engine.import();
+ {
+ Db db2("snapshot");
+ assert(equal(db, db2));
+ }
+ }
+
+ // ******************************* TEST 3 ********************************//
+ {
+ std::cout << "TEST3" << std::endl;
+ // add vertexs
+ // add edges
+ // make snapshot of db
+ // compare database with different named database
+ Db db("snapshot", false);
+ run(cvl_n, create_vertex_label, db);
+ add_edge(cvl_n, db);
+ db.snap_engine.make_snapshot();
+ {
+ Db db2("not_snapshot");
+ assert(!equal(db, db2));
+ }
+ }
+
+ // ******************************* TEST 4 ********************************//
+ {
+ std::cout << "TEST4" << std::endl;
+ // add vertices LABEL
+ // add properties
+ // add vertices LABEL
+ // add index on proprety
+ // assert index containts vertices
+ // make snapshot
+ // create new db
+ // assert index on LABEL in new db exists
+ // assert index in new db containts vertice
+ Db db("snapshot", false);
+ run(cvl_n, create_vertex_label, db);
+ auto &family = db.graph.vertices.property_family_find_or_create("prop");
+ add_property_different_int(db, family);
+ run(cvl_n, create_vertex_other, db);
+ IndexDefinition idef = {
+ IndexLocation{VertexSide, Option<std::string>("prop"),
+ Option<std::string>(), Option<std::string>()},
+ IndexType{false, None}};
+ assert(db.indexes().add_index(idef));
+ assert(cvl_n == size(db, family.index));
+ db.snap_engine.make_snapshot();
+ {
+ Db db2("snapshot");
+ assert(cvl_n == size(db, db2.graph.vertices
+ .property_family_find_or_create("prop")
+ .index));
+ }
}
// TODO: more tests
From a6f72dc3fd533f5c594376db4c65d40dee4a2ec0 Mon Sep 17 00:00:00 2001
From: Kruno Tomola Fabro <krunotf@memgraph.io>
Date: Tue, 13 Sep 2016 17:53:40 +0100
Subject: [PATCH 3/5] Added stress test for ConncurentList.
---
poc/add_double.cpp | 5 ++
src/snapshot/snapshot_engine.cpp | 3 +-
tests/concurrent/common.h | 22 ++++++++
tests/concurrent/conncurent_list.cpp | 77 ++++++++++++++++++++++++++++
4 files changed, 106 insertions(+), 1 deletion(-)
create mode 100644 tests/concurrent/conncurent_list.cpp
diff --git a/poc/add_double.cpp b/poc/add_double.cpp
index 4c003ede1..5c0dd003b 100644
--- a/poc/add_double.cpp
+++ b/poc/add_double.cpp
@@ -27,6 +27,11 @@ void add_scores(Db &db, double max_value, std::string const &property_name)
t.commit();
}
+// Tool to add double propertys to all vertices.
+// // Accepts flags for csv import.
+// -db name , will create database with that name.
+// -pn name , will name property with that name,default: name=score.
+// -max number , will set range of property [0,max], default: max=1
int main(int argc, char **argv)
{
logging::init_async();
diff --git a/src/snapshot/snapshot_engine.cpp b/src/snapshot/snapshot_engine.cpp
index 8f5fe0449..70d2fcd8e 100644
--- a/src/snapshot/snapshot_engine.cpp
+++ b/src/snapshot/snapshot_engine.cpp
@@ -292,7 +292,8 @@ std::string SnapshotEngine::snapshot_file(std::time_t const &now,
auto now_nano = std::chrono::time_point_cast<std::chrono::nanoseconds>(
std::chrono::high_resolution_clock::now())
.time_since_epoch()
- .count();
+ .count() %
+ (1000 * 1000 * 1000);
return snapshot_db_dir() + "/" + std::to_string(now) + "_" +
std::to_string(now_nano) + "_" + type;
}
diff --git a/tests/concurrent/common.h b/tests/concurrent/common.h
index 82910cdd4..e32926251 100644
--- a/tests/concurrent/common.h
+++ b/tests/concurrent/common.h
@@ -13,6 +13,7 @@
#include "data_structures/concurrent/concurrent_multiset.hpp"
#include "data_structures/concurrent/concurrent_set.hpp"
#include "data_structures/concurrent/skiplist.hpp"
+#include "data_structures/concurrent/concurrent_list.hpp"
#include "data_structures/static_array.hpp"
#include "utils/assert.hpp"
#include "logging/default.hpp"
@@ -64,6 +65,26 @@ void check_present_same(typename S::Accessor &acc,
// Checks if reported size and traversed size are equal to given size.
template <typename S>
+void check_size_list(S &acc, long long size)
+{
+ // check size
+
+ permanent_assert(acc.size() == size, "Size should be " << size
+ << ", but size is "
+ << acc.size());
+
+ // check count
+
+ size_t iterator_counter = 0;
+
+ for (auto elem : acc) {
+ ++iterator_counter;
+ }
+ permanent_assert(iterator_counter == size, "Iterator count should be "
+ << size << ", but size is "
+ << iterator_counter);
+}
+template <typename S>
void check_size(typename S::Accessor &acc, long long size)
{
// check size
@@ -83,6 +104,7 @@ void check_size(typename S::Accessor &acc, long long size)
<< size << ", but size is "
<< iterator_counter);
}
+
// Checks if order in list is maintened. It expects map
template <typename S>
void check_order(typename S::Accessor &acc)
diff --git a/tests/concurrent/conncurent_list.cpp b/tests/concurrent/conncurent_list.cpp
new file mode 100644
index 000000000..ab6bff71e
--- /dev/null
+++ b/tests/concurrent/conncurent_list.cpp
@@ -0,0 +1,77 @@
+#include "common.h"
+
+#define THREADS_NO 8
+constexpr size_t key_range = 1e2;
+constexpr size_t op_per_thread = 1e5;
+// Depending on value there is a possiblity of numerical overflow
+constexpr size_t max_number = 10;
+constexpr size_t no_find_per_change = 2;
+constexpr size_t no_insert_for_one_delete = 1;
+
+// This test simulates behavior of transactions.
+// Each thread makes a series of finds interleaved with method which change.
+// Exact ratio of finds per change and insert per delete can be regulated with
+// no_find_per_change and no_insert_for_one_delete.
+int main()
+{
+ init_log();
+ memory_check(THREADS_NO, [] {
+ ConcurrentList<std::pair<int, int>> list;
+
+ auto futures = run<std::pair<long long, long long>>(
+ THREADS_NO, [&](auto index) mutable {
+ auto rand = rand_gen(key_range);
+ auto rand_change = rand_gen_bool(no_find_per_change);
+ auto rand_delete = rand_gen_bool(no_insert_for_one_delete);
+ long long sum = 0;
+ long long count = 0;
+
+ for (int i = 0; i < op_per_thread; i++) {
+ auto num = rand();
+ auto data = num % max_number;
+ if (rand_change()) {
+ if (rand_delete()) {
+ for (auto it = list.begin(); it != list.end();
+ it++) {
+ if (it->first == num) {
+ if (it.remove()) {
+ sum -= data;
+ count--;
+ }
+ break;
+ }
+ }
+ } else {
+ list.begin().push(std::make_pair(num, data));
+ sum += data;
+ count++;
+ }
+ } else {
+ for (auto &v : list) {
+ if (v.first == num) {
+ permanent_assert(v.second == data,
+ "Data is invalid");
+ break;
+ }
+ }
+ }
+ }
+
+ return std::pair<long long, long long>(sum, count);
+ });
+
+ auto it = list.begin();
+ long long sums = 0;
+ long long counters = 0;
+ for (auto &data : collect(futures)) {
+ sums += data.second.first;
+ counters += data.second.second;
+ }
+
+ for (auto &e : list) {
+ sums -= e.second;
+ }
+ permanent_assert(sums == 0, "Same values aren't present");
+ check_size_list<ConcurrentList<std::pair<int, int>>>(list, counters);
+ });
+}
From 9bd845a62e08adc2db13c264c02f62fca87732ab Mon Sep 17 00:00:00 2001
From: Kruno Tomola Fabro <krunotf@memgraph.io>
Date: Tue, 13 Sep 2016 18:08:34 +0100
Subject: [PATCH 4/5] Added max_no_threads. It is meant for easily change no
threads in concurrent tests.
---
tests/concurrent/common.h | 3 +++
tests/concurrent/conncurent_list.cpp | 2 +-
tests/concurrent/dynamic_bitset.cpp | 2 +-
tests/concurrent/dynamic_bitset_clear_n.cpp | 2 +-
tests/concurrent/dynamic_bitset_set.cpp | 2 +-
tests/concurrent/dynamic_bitset_set_n.cpp | 2 +-
tests/concurrent/sl_insert.cpp | 2 +-
tests/concurrent/sl_insert_competetive.cpp | 2 +-
tests/concurrent/sl_map.cpp | 2 +-
tests/concurrent/sl_memory.cpp | 2 +-
tests/concurrent/sl_memory_leak.cpp | 2 +-
tests/concurrent/sl_multiiterator.cpp | 2 +-
tests/concurrent/sl_multiiterator_remove.cpp | 2 +-
tests/concurrent/sl_multiiterator_remove_duplicates.cpp | 2 +-
tests/concurrent/sl_multimap.cpp | 2 +-
tests/concurrent/sl_multiset.cpp | 2 +-
tests/concurrent/sl_remove_competetive.cpp | 2 +-
tests/concurrent/sl_remove_disjoint.cpp | 2 +-
tests/concurrent/sl_remove_joint.cpp | 2 +-
tests/concurrent/sl_set.cpp | 2 +-
tests/concurrent/sl_simulation.cpp | 2 +-
tests/integration/snapshot.cpp | 1 -
tests/poc/profile.cpp | 0
23 files changed, 23 insertions(+), 21 deletions(-)
delete mode 100644 tests/poc/profile.cpp
diff --git a/tests/concurrent/common.h b/tests/concurrent/common.h
index e32926251..d43f7976b 100644
--- a/tests/concurrent/common.h
+++ b/tests/concurrent/common.h
@@ -20,6 +20,9 @@
#include "logging/streams/stdout.hpp"
#include "utils/sysinfo/memory.hpp"
+// Sets max number of threads that will be used in concurrent tests.
+constexpr int max_no_threads=8;
+
using std::cout;
using std::endl;
using map_t = ConcurrentMap<int, int>;
diff --git a/tests/concurrent/conncurent_list.cpp b/tests/concurrent/conncurent_list.cpp
index ab6bff71e..aeea7bfda 100644
--- a/tests/concurrent/conncurent_list.cpp
+++ b/tests/concurrent/conncurent_list.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t key_range = 1e2;
constexpr size_t op_per_thread = 1e5;
// Depending on value there is a possiblity of numerical overflow
diff --git a/tests/concurrent/dynamic_bitset.cpp b/tests/concurrent/dynamic_bitset.cpp
index 252874650..ac24109ea 100644
--- a/tests/concurrent/dynamic_bitset.cpp
+++ b/tests/concurrent/dynamic_bitset.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t op_per_thread = 1e5;
constexpr size_t bit_part_len = 2;
constexpr size_t no_slots = 1e4;
diff --git a/tests/concurrent/dynamic_bitset_clear_n.cpp b/tests/concurrent/dynamic_bitset_clear_n.cpp
index 61b5384aa..6f38bbf64 100644
--- a/tests/concurrent/dynamic_bitset_clear_n.cpp
+++ b/tests/concurrent/dynamic_bitset_clear_n.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 4
+constexpr size_t THREADS_NO = std::min(max_no_threads, 4);
constexpr size_t op_per_thread = 1e5;
constexpr size_t up_border_bit_set_pow2 = 3;
constexpr size_t key_range =
diff --git a/tests/concurrent/dynamic_bitset_set.cpp b/tests/concurrent/dynamic_bitset_set.cpp
index f219a6a97..b1ec1eae8 100644
--- a/tests/concurrent/dynamic_bitset_set.cpp
+++ b/tests/concurrent/dynamic_bitset_set.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t op_per_thread = 1e5;
constexpr size_t key_range = op_per_thread * THREADS_NO * 3;
diff --git a/tests/concurrent/dynamic_bitset_set_n.cpp b/tests/concurrent/dynamic_bitset_set_n.cpp
index c05118feb..b31bcda2a 100644
--- a/tests/concurrent/dynamic_bitset_set_n.cpp
+++ b/tests/concurrent/dynamic_bitset_set_n.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 4
+constexpr size_t THREADS_NO = std::min(max_no_threads, 4);
constexpr size_t op_per_thread = 1e5;
constexpr size_t up_border_bit_set_pow2 = 3;
constexpr size_t key_range =
diff --git a/tests/concurrent/sl_insert.cpp b/tests/concurrent/sl_insert.cpp
index a0b83e18f..7948c2291 100644
--- a/tests/concurrent/sl_insert.cpp
+++ b/tests/concurrent/sl_insert.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t elems_per_thread = 100000;
constexpr size_t key_range = elems_per_thread * THREADS_NO * 2;
diff --git a/tests/concurrent/sl_insert_competetive.cpp b/tests/concurrent/sl_insert_competetive.cpp
index 8050a377c..636ca9264 100644
--- a/tests/concurrent/sl_insert_competetive.cpp
+++ b/tests/concurrent/sl_insert_competetive.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t elems_per_thread = 100000;
constexpr size_t key_range = elems_per_thread * THREADS_NO * 2;
diff --git a/tests/concurrent/sl_map.cpp b/tests/concurrent/sl_map.cpp
index 8b7684073..c56a1aa2f 100644
--- a/tests/concurrent/sl_map.cpp
+++ b/tests/concurrent/sl_map.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t elems_per_thread = 1e5;
int main()
diff --git a/tests/concurrent/sl_memory.cpp b/tests/concurrent/sl_memory.cpp
index cc7522a3b..f69bd66a5 100644
--- a/tests/concurrent/sl_memory.cpp
+++ b/tests/concurrent/sl_memory.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t elements = 2e6;
diff --git a/tests/concurrent/sl_memory_leak.cpp b/tests/concurrent/sl_memory_leak.cpp
index c5e384ee5..6c7bf64da 100644
--- a/tests/concurrent/sl_memory_leak.cpp
+++ b/tests/concurrent/sl_memory_leak.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 1
+constexpr size_t THREADS_NO = std::min(max_no_threads, 1);
constexpr size_t elems_per_thread = 16e5;
// Known memory leak at 1,600,000 elements.
diff --git a/tests/concurrent/sl_multiiterator.cpp b/tests/concurrent/sl_multiiterator.cpp
index ebff8c0ca..68bde7fbc 100644
--- a/tests/concurrent/sl_multiiterator.cpp
+++ b/tests/concurrent/sl_multiiterator.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t key_range = 1e4;
constexpr size_t op_per_thread = 1e5;
// Depending on value there is a possiblity of numerical overflow
diff --git a/tests/concurrent/sl_multiiterator_remove.cpp b/tests/concurrent/sl_multiiterator_remove.cpp
index e8b8e3d07..228788e7f 100644
--- a/tests/concurrent/sl_multiiterator_remove.cpp
+++ b/tests/concurrent/sl_multiiterator_remove.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t key_range = 1e4;
constexpr size_t op_per_thread = 1e5;
// Depending on value there is a possiblity of numerical overflow
diff --git a/tests/concurrent/sl_multiiterator_remove_duplicates.cpp b/tests/concurrent/sl_multiiterator_remove_duplicates.cpp
index da2215ec0..01712d199 100644
--- a/tests/concurrent/sl_multiiterator_remove_duplicates.cpp
+++ b/tests/concurrent/sl_multiiterator_remove_duplicates.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 4
+constexpr size_t THREADS_NO = std::min(max_no_threads, 4);
constexpr size_t key_range = 1e4;
constexpr size_t op_per_thread = 1e5;
// Depending on value there is a possiblity of numerical overflow
diff --git a/tests/concurrent/sl_multimap.cpp b/tests/concurrent/sl_multimap.cpp
index 5953a4ca5..9c1fa97ba 100644
--- a/tests/concurrent/sl_multimap.cpp
+++ b/tests/concurrent/sl_multimap.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t key_range = 1e4;
constexpr size_t op_per_thread = 1e5;
// Depending on value there is a possiblity of numerical overflow
diff --git a/tests/concurrent/sl_multiset.cpp b/tests/concurrent/sl_multiset.cpp
index eaca13ca0..9a5bdd006 100644
--- a/tests/concurrent/sl_multiset.cpp
+++ b/tests/concurrent/sl_multiset.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t key_range = 1e4;
constexpr size_t op_per_thread = 1e5;
constexpr size_t no_insert_for_one_delete = 1;
diff --git a/tests/concurrent/sl_remove_competetive.cpp b/tests/concurrent/sl_remove_competetive.cpp
index 614d00d46..13a7967da 100644
--- a/tests/concurrent/sl_remove_competetive.cpp
+++ b/tests/concurrent/sl_remove_competetive.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t op_per_thread = 1e5;
// Depending on value there is a possiblity of numerical overflow
constexpr size_t max_number = 10;
diff --git a/tests/concurrent/sl_remove_disjoint.cpp b/tests/concurrent/sl_remove_disjoint.cpp
index 031d5106e..99b9465ca 100644
--- a/tests/concurrent/sl_remove_disjoint.cpp
+++ b/tests/concurrent/sl_remove_disjoint.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t key_range = 1e5;
constexpr size_t op_per_thread = 1e6;
constexpr size_t no_insert_for_one_delete = 1;
diff --git a/tests/concurrent/sl_remove_joint.cpp b/tests/concurrent/sl_remove_joint.cpp
index b32bae485..495433966 100644
--- a/tests/concurrent/sl_remove_joint.cpp
+++ b/tests/concurrent/sl_remove_joint.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t key_range = 1e4;
constexpr size_t op_per_thread = 1e5;
// Depending on value there is a possiblity of numerical overflow
diff --git a/tests/concurrent/sl_set.cpp b/tests/concurrent/sl_set.cpp
index e86d0a6ae..84c6c582c 100644
--- a/tests/concurrent/sl_set.cpp
+++ b/tests/concurrent/sl_set.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t key_range = 1e4;
constexpr size_t op_per_thread = 1e5;
constexpr size_t no_insert_for_one_delete = 2;
diff --git a/tests/concurrent/sl_simulation.cpp b/tests/concurrent/sl_simulation.cpp
index 564d3d1eb..ffeb11f2c 100644
--- a/tests/concurrent/sl_simulation.cpp
+++ b/tests/concurrent/sl_simulation.cpp
@@ -1,6 +1,6 @@
#include "common.h"
-#define THREADS_NO 8
+constexpr size_t THREADS_NO = std::min(max_no_threads, 8);
constexpr size_t key_range = 1e5;
constexpr size_t op_per_thread = 1e6;
// Depending on value there is a possiblity of numerical overflow
diff --git a/tests/integration/snapshot.cpp b/tests/integration/snapshot.cpp
index c68c1a5bd..50975722d 100644
--- a/tests/integration/snapshot.cpp
+++ b/tests/integration/snapshot.cpp
@@ -188,7 +188,6 @@ int main(void)
// empty database
// import snapshot
// assert database empty
-
Db db("snapshot", false);
db.snap_engine.make_snapshot();
run(cvl_n, create_vertex_label, db);
diff --git a/tests/poc/profile.cpp b/tests/poc/profile.cpp
deleted file mode 100644
index e69de29bb..000000000
From c75382cb02dc2c7b83f9264fd720d430388fc4ef Mon Sep 17 00:00:00 2001
From: Kruno Tomola Fabro <krunotf@memgraph.io>
Date: Wed, 14 Sep 2016 12:19:56 +0100
Subject: [PATCH 5/5] Tmp commit.
---
include/storage/indexes/indexes.hpp | 57 ++++++-
tests/CMakeLists.txt | 9 ++
tests/integration/index.cpp | 240 ++++++++++++++++++++++++++++
3 files changed, 305 insertions(+), 1 deletion(-)
create mode 100644 tests/integration/index.cpp
diff --git a/include/storage/indexes/indexes.hpp b/include/storage/indexes/indexes.hpp
index a385223fb..153d306b0 100644
--- a/include/storage/indexes/indexes.hpp
+++ b/include/storage/indexes/indexes.hpp
@@ -20,6 +20,61 @@ public:
// Adds index defined in given definition. Returns true if successfull.
bool add_index(IndexDefinition id);
+ //
+ // // Returns index from location.
+ // template <class TG, class K>
+ // Option<IndexHolder<TG, K>> get_index(IndexLocation loc)
+ // {
+ // size_t code = loc.location_code();
+ //
+ // switch (code) {
+ // case 0: // Illegal location
+ // return Option<IndexHolder<TG, K>>();
+ //
+ // case 1:
+ // switch (loc.side) {
+ // case EdgeSide: {
+ // return make_option(
+ // db.graph.edges
+ // .property_family_find_or_create(loc.property_name.get())
+ // .index);
+ // }
+ // case VertexSide: {
+ // return make_option(
+ // db.graph.vertices
+ // .property_family_find_or_create(loc.property_name.get())
+ // .index);
+ // }
+ // default:
+ // throw new NonExhaustiveSwitch("Unkown side: " +
+ // std::to_string(loc.side));
+ // };
+ //
+ // case 2: // Can't be removed
+ // return Option<IndexHolder<TG, K>>();
+ //
+ // case 3: // Not yet implemented
+ // throw new NotYetImplemented("Getting index over label and "
+ // "property isn't yet implemented");
+ // case 4: // Can't be removed
+ // return Option<IndexHolder<TG, K>>();
+ //
+ // case 5: // Not yet implemented
+ // throw new NotYetImplemented("Getting index over edge_type and "
+ // "property isn't yet implemented");
+ // case 6: // Not yet implemented
+ // throw new NotYetImplemented("Getting index over edge_type and "
+ // "label isn't yet implemented");
+ // case 7: // Not yet implemented
+ // throw new NotYetImplemented("Getting index over label, edge_type
+ // "
+ // "and property isn't yet
+ // implemented");
+ // default:
+ // throw new NonExhaustiveSwitch("Unkown index location code: " +
+ // std::to_string(code));
+ // }
+ // }
// Removes index from given location. Returns true if successfull or if no
// index was present. False if index location is illegal.
@@ -98,7 +153,7 @@ public:
f);
}
- // Updates property indexes for given TypeGroup TG and IU index update
+ // Updates property indexes for given TypeGroup TG and IU index_update
template <class TG, class IU>
bool update_property_indexes(IU &iu, const tx::Transaction &t)
{
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 16d377a87..5e8a88c40 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -89,6 +89,15 @@ target_link_libraries(snapshot ${yaml_static_lib})
add_test(NAME snapshot COMMAND snapshot)
set_property(TARGET snapshot PROPERTY CXX_STANDARD 14)
+# test index validity
+add_executable(index integration/index.cpp)
+target_link_libraries(index memgraph)
+target_link_libraries(index Threads::Threads)
+target_link_libraries(index ${fmt_static_lib})
+target_link_libraries(index ${yaml_static_lib})
+add_test(NAME index COMMAND index)
+set_property(TARGET index PROPERTY CXX_STANDARD 14)
+
# test query engine
add_executable(integration_query_engine integration/query_engine.cpp)
target_link_libraries(integration_query_engine Threads::Threads)
diff --git a/tests/integration/index.cpp b/tests/integration/index.cpp
new file mode 100644
index 000000000..1f9790f02
--- /dev/null
+++ b/tests/integration/index.cpp
@@ -0,0 +1,240 @@
+#include "query_engine/hardcode/queries.hpp"
+
+#include <random>
+
+#include "barrier/barrier.cpp"
+
+#include "logging/default.hpp"
+#include "logging/streams/stdout.hpp"
+#include "query_engine/query_stripper.hpp"
+#include "storage/indexes/indexes.hpp"
+#include "utils/sysinfo/memory.hpp"
+
+// Returns uniform random size_t generator from range [0,n>
+auto rand_gen(size_t n)
+{
+ std::default_random_engine generator;
+ std::uniform_int_distribution<size_t> distribution(0, n - 1);
+ return std::bind(distribution, generator);
+}
+
+void run(size_t n, std::string &query, Db &db)
+{
+ auto stripper = make_query_stripper(TK_LONG, TK_FLOAT, TK_STR, TK_BOOL);
+ auto qf = load_queries(barrier::trans(db));
+ auto stripped = stripper.strip(query);
+ std::cout << "Running query [" << stripped.hash << "] for " << n << " time."
+ << std::endl;
+ for (int i = 0; i < n; i++) {
+ properties_t vec = stripped.arguments;
+ assert(qf[stripped.hash](std::move(vec)));
+ }
+}
+
+void add_edge(size_t n, Db &db)
+{
+ auto stripper = make_query_stripper(TK_LONG, TK_FLOAT, TK_STR, TK_BOOL);
+ auto qf = load_queries(barrier::trans(db));
+ std::string query = "MATCH (n1), (n2) WHERE ID(n1)=0 AND "
+ "ID(n2)=1 CREATE (n1)<-[r:IS {age: "
+ "25,weight: 70}]-(n2) RETURN r";
+
+ auto stripped = stripper.strip(query);
+ std::cout << "Running query [" << stripped.hash << "] for " << n
+ << " time to add edge." << std::endl;
+
+ std::vector<int64_t> vertices;
+ for (auto &v : db.graph.vertices.access()) {
+ vertices.push_back(v.second.id);
+ }
+
+ auto rand = rand_gen(vertices.size());
+ for (int i = 0; i < n; i++) {
+ properties_t vec = stripped.arguments;
+ vec[0] = Property(Int64(vertices[rand()]), Flags::Int64);
+ vec[1] = Property(Int64(vertices[rand()]), Flags::Int64);
+ assert(qf[stripped.hash](std::move(vec)));
+ }
+}
+
+void add_property(Db &db, StoredProperty<TypeGroupVertex> &prop)
+{
+ DbAccessor t(db);
+
+ t.vertex_access().fill().for_all([&](auto va) { va.set(prop); });
+
+ assert(t.commit());
+}
+
+void add_vertex_property_serial_int(Db &db, PropertyFamily<TypeGroupVertex> &f)
+{
+ DbAccessor t(db);
+
+ auto key = f.get(Int64::type).family_key();
+
+ size_t i = 0;
+ t.vertex_access().fill().for_all([&](auto va) mutable {
+ va.set(StoredProperty<TypeGroupVertex>(Int64(i), key));
+ i++;
+ });
+
+ assert(t.commit());
+}
+
+void add_edge_property_serial_int(Db &db, PropertyFamily<TypeGroupEdge> &f)
+{
+ DbAccessor t(db);
+
+ auto key = f.get(Int64::type).family_key();
+
+ size_t i = 0;
+ t.edge_access().fill().for_all([&](auto va) mutable {
+ va.set(StoredProperty<TypeGroupEdge>(Int64(i), key));
+ i++;
+ });
+
+ assert(t.commit());
+}
+
+template <class TG>
+size_t size(Db &db, IndexHolder<TG, std::nullptr_t> &h)
+{
+ DbAccessor t(db);
+
+ size_t count = 0;
+ auto oin = h.get_read();
+ if (oin.is_present()) {
+ oin.get()->for_range(t).for_all([&](auto va) mutable { count++; });
+ }
+
+ t.commit();
+
+ return count;
+}
+
+void assert_empty(Db &db)
+{
+ assert(db.graph.vertices.access().size() == 0);
+ assert(db.graph.edges.access().size() == 0);
+}
+
+void clean_vertex(Db &db)
+{
+ DbTransaction t(db);
+ t.clean_vertex_section();
+ t.trans.commit();
+}
+
+void clean_edge(Db &db)
+{
+ DbTransaction t(db);
+ t.clean_edge_section();
+ t.trans.commit();
+}
+
+void clear_database(Db &db)
+{
+ std::string delete_all_vertices = "MATCH (n) DELETE n";
+ std::string delete_all_edges = "MATCH ()-[r]-() DELETE r";
+
+ run(1, delete_all_edges, db);
+ run(1, delete_all_vertices, db);
+ clean_vertex(db);
+ clean_edge(db);
+ assert_empty(db);
+}
+
+bool equal(Db &a, Db &b)
+{
+ {
+ auto acc_a = a.graph.vertices.access();
+ auto acc_b = b.graph.vertices.access();
+
+ if (acc_a.size() != acc_b.size()) {
+ return false;
+ }
+
+ auto it_a = acc_a.begin();
+ auto it_b = acc_b.begin();
+
+ for (auto i = acc_a.size(); i > 0; i--) {
+ // TODO: compare
+ }
+ }
+
+ {
+ auto acc_a = a.graph.edges.access();
+ auto acc_b = b.graph.edges.access();
+
+ if (acc_a.size() != acc_b.size()) {
+ return false;
+ }
+
+ auto it_a = acc_a.begin();
+ auto it_b = acc_b.begin();
+
+ for (auto i = acc_a.size(); i > 0; i--) {
+ // TODO: compare
+ }
+ }
+
+ return true;
+}
+
+int main(void)
+{
+ logging::init_async();
+ logging::log->pipe(std::make_unique<Stdout>());
+
+ size_t cvl_n = 1000;
+
+ std::string create_vertex_label =
+ "CREATE (n:LABEL {name: \"cleaner_test\"}) RETURN n";
+ std::string create_vertex_other =
+ "CREATE (n:OTHER {name: \"cleaner_test\"}) RETURN n";
+ std::string delete_label_vertices = "MATCH (n:LABEL) DELETE n";
+ std::string delete_all_vertices = "MATCH (n) DELETE n";
+
+ IndexDefinition vertex_property_nonunique_unordered = {
+ IndexLocation{VertexSide, Option<std::string>("prop"),
+ Option<std::string>(), Option<std::string>()},
+ IndexType{false, None}};
+ IndexDefinition edge_property_nonunique_unordered = {
+ IndexLocation{EdgeSide, Option<std::string>("prop"),
+ Option<std::string>(), Option<std::string>()},
+ IndexType{false, None}};
+ IndexDefinition edge_property_unique_ordered = {
+ IndexLocation{EdgeSide, Option<std::string>("prop"),
+ Option<std::string>(), Option<std::string>()},
+ IndexType{true, Ascending}};
+ IndexDefinition vertex_property_unique_ordered = {
+ IndexLocation{VertexSide, Option<std::string>("prop"),
+ Option<std::string>(), Option<std::string>()},
+ IndexType{true, Ascending}};
+
+ // ******************************* TEST 1 ********************************//
+ {
+ std::cout << "TEST1" << std::endl;
+ // add indexes
+ // add vertices LABEL
+ // add edges
+ // add vertices property
+ // assert index size.
+ Db db("index", false);
+ assert(db.indexes().add_index(vertex_property_nonunique_unordered));
+ assert(db.indexes().add_index(edge_property_nonunique_unordered));
+ run(cvl_n, create_vertex_label, db);
+ add_edge(cvl_n, db);
+ assert(cvl_n ==
+ size(db, db.graph.vertices.property_family_find_or_create("prop")
+ .index));
+ assert(
+ cvl_n ==
+ size(db,
+ db.graph.edges.property_family_find_or_create("prop").index));
+ }
+
+ // TODO: more tests
+
+ return 0;
+}