query engine code, labels (unfinished), TODO: query engine module system

CMakeLists.txt

@@ -116,7 +116,7 @@ foreach(source_folder ${source_folders})
 endforeach()
 
 # compiler options
-SET(COMPILE_OPTIONS "-O0 -g3 -Wall -Werror -fmessage-length=0")
+SET(COMPILE_OPTIONS "-O2 -Wall -Werror -fmessage-length=0")
 
 # add all cpp file recursive into sourceFiles varibale
 # FILE(GLOB_RECURSE sourceFiles ${src_dir}/*.cpp)
@@ -152,30 +152,38 @@ include_directories(${r3_source_dir}/include)
 # creates build/libcypher_lib.a
 add_library(cypher_lib STATIC ${CMAKE_BINARY_DIR}/cypher.cpp)
 
-# tests
-enable_testing()
-add_subdirectory(tests)
-
 # REST API preprocessor
 EXECUTE_PROCESS(
     COMMAND python link_resources.py
     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/api
 )
 
-# memgraph executable
-add_executable(memgraph src/memgraph.cpp)
-add_dependencies(memgraph cypher_lib)
-# memgraph link libraries
-target_link_libraries(memgraph Threads::Threads)
-target_link_libraries(memgraph pcre)
-target_link_libraries(memgraph ${libuv_static_lib})
-target_link_libraries(memgraph ${r3_static_lib})
-target_link_libraries(memgraph ${http_parser_static_lib})
+# # memgraph executable
+# add_executable(memgraph src/memgraph.cpp)
+# add_dependencies(memgraph cypher_lib)
+# # memgraph link libraries
+# target_link_libraries(memgraph Threads::Threads)
+# target_link_libraries(memgraph pcre)
+# target_link_libraries(memgraph ${libuv_static_lib})
+# target_link_libraries(memgraph ${r3_static_lib})
+# target_link_libraries(memgraph ${http_parser_static_lib})
 
-# query_engine executable
-add_executable(query_engine src/query_engine/main.cpp)
-# query_engine link libraries
-target_link_libraries(query_engine Threads::Threads)
-target_link_libraries(query_engine dl)
-target_link_libraries(query_engine cypher_lib)
-target_link_libraries(query_engine ${fmt_static_lib})
+# # query_engine executable
+# add_executable(query_engine src/query_engine/main_query_engine.cpp)
+# # query_engine link libraries
+# target_link_libraries(query_engine Threads::Threads)
+# target_link_libraries(query_engine dl)
+# target_link_libraries(query_engine cypher_lib)
+# target_link_libraries(query_engine ${fmt_static_lib})
+
+# # query hasher executable
+# add_executable(query_hasher src/query_engine/main_query_hasher.cpp)
+# target_link_libraries(query_hasher ${fmt_static_lib})
+# 
+# # hard coded implementation of queries
+# add_executable(queries src/query_engine/main_queries.cpp)
+# target_link_libraries(queries ${fmt_static_lib})
+
+# tests
+enable_testing()
+add_subdirectory(tests)

src/cypher/cypher.y

@@ -13,6 +13,7 @@
 
 %syntax_error
 {
+#ifndef NDEBUG
     int n = sizeof(yyTokenName) / sizeof(yyTokenName[0]);
     for (int i = 0; i < n; ++i) {
         int a = yy_find_shift_action(yypParser, (YYCODETYPE)i);
@@ -21,6 +22,7 @@
         }
     }
     throw SyntaxError(TOKEN->value);
+#endif
 }
 
 %stack_overflow

src/data_structures/skiplist/skiplist.hpp

@@ -453,7 +453,7 @@ public:
         return Accessor(this);
     }
 
-    Accessor access() const
+    const Accessor access() const
     {
         return Accessor(this);
     }

src/data_structures/skiplist/skiplist_gc.hpp

@@ -28,18 +28,24 @@ public:
             }
         }
 
-        long long counter = 0;
-
-        // destroy all elements from local_freelist
-        for (auto element : local_freelist) {
-            counter++;
-            if (element->flags.is_marked()) T::destroy(element);
+        if (local_freelist.size() > 0) {
+            std::cout << "GC started" <<  std::endl;
+            std::cout << "Local skiplist size: " <<
+                         local_freelist.size() << std::endl;
+            long long counter = 0;
+            // destroy all elements from local_freelist
+            for (auto element : local_freelist) {
+                counter++;
+                if (element->flags.is_marked()) T::destroy(element);
+            }
+            std::cout << "Number of destroyed elements " << counter << std::endl;
         }
-
-        std::cout << "Destroy has been called: " << counter << std::endl;
     }
 
-    void collect(T *node) { freelist.add(node); }
+    void collect(T *node) 
+    {
+        freelist.add(node);
+    }
 
 private:
     FreeList<T> freelist;

src/memory/freelist.hpp

@@ -17,6 +17,11 @@ public:
         data.emplace_back(element);
     }
 
+    size_t size() const
+    {
+        return data.size();
+    }
+
 private:
     std::vector<T *> data;
 };

src/query_engine/compile.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-# TODO: create Makefile or cmake script
-# cd compiled/cpu
-# clang++ -std=c++1y create_return.cpp -o create_return.so -I../../../ -shared -fPIC
-# cd ../..
-# clang++ -std=c++1y -g -I../ -I ../lib/yaml-cpp/include main.cpp ../cypher/cypher.cpp -o engine -L ../lib/yaml-cpp/build -l yaml-cpp -l dl
-# clang++ -std=c++1y -g -O2 -I../ main.cpp ../cypher/cypher.cpp -o engine.out -l dl -pthread
-clang++ -std=c++1y -g -I../ main.cpp ../cypher/cypher.cpp -o engine.out -l dl -pthread

src/query_engine/main_queries.cpp

@@ -0,0 +1,108 @@
+#include <iostream>
+
+#include "database/db.hpp"
+#include "query_stripper.hpp"
+#include "storage/model/properties/property.hpp"
+#include "utils/command_line/arguments.hpp"
+#include "cypher/common.hpp"
+#include "utils/time/timer.hpp"
+
+// --
+// DESCRIPTION: create account
+// FULL:        CREATE (n:ACCOUNT {id: 50, name: "Nikola", country: "Croatia", created_at: 14634563}) RETURN n
+// STRIPPED:    CREATE(n:ACCOUNT{id:0,name:1,country:2,created_at:3})RETURNn
+// HASH:        10597108978382323595
+// --
+// DESCRIPTION: create personnel
+// FULL:        CREATE (n:PERSONNEL {id: 23, role: "CTO", created_at: 1235345}) RETURN n
+// STRIPPED:    CREATE(n:PERSONNEL{id:0,role:1,created_at:2})RETURNn
+// HASH:        4037885257628527960 
+// --
+// DESCRIPTION: create edge between ACCOUNT node and PERSONNEL node (IS type)
+// FULL:        MATCH (a:ACCOUNT {id:50}), (p:PERSONNEL {id: 23}) CREATE (a)-[:IS]->(p) 
+// STRIPPED:    MATCH(a:ACCOUNT{id:0}),(p:PERSONNEL{id:1})CREATE(a)-[:IS]->(p)
+// HASH:        16888190822925675190 
+// --
+// DESCRIPTION: find ACCOUNT node, PERSONNEL node and edge between them
+// FULL:        MATCH (a:ACCOUNT {id:50})-[r:IS]->(p:PERSONNEL {id: 23}) RETURN a,r,p 
+// STRIPPED:    MATCH(a:ACCOUNT{id:0})-[r:IS]->(p:PERSONNEL{id:1})RETURNa,r,p
+// HASH:        9672752533852902744 
+// --
+// DESCRIPTION: create OPPORTUNITY
+// FULL:         
+// STRIPPED:   
+// HASH:       
+// --
+// DESCRIPTION: create PERSONNEL-[:CREATED]->OPPORTUNITY
+// FULL:         
+// STRIPPED:   
+// HASH:       
+// --
+// DESCRIPTION: create COMPANY
+// FULL:         
+// STRIPPED:   
+// HASH:       
+// --
+// DESCRIPTION: create OPPORTUNITY-[:MATCH]->COMPANY
+// FULL:         
+// STRIPPED:   
+// HASH:       
+// --
+// DESCRIPTION: create edge between two nodes found by the ID
+// DESCRIPTION: fine node by the ID
+// DESCRIPTION: find edge by the ID
+// DESCRIPTION: find shortest path between two nodes specified by ID
+
+int main(int argc, char** argv)
+{
+    Db db;
+    std::map<uint64_t, std::function<void(const properties_t&)>> queries;
+
+    auto create_account = [&db](const properties_t& args)
+    {
+        auto& t = db.tx_engine.begin();
+        auto vertex_accessor = db.graph.vertices.insert(t);
+        vertex_accessor.property(
+            "id", args[0]
+        );
+        vertex_accessor.property(
+            "name", args[1]
+        );
+        vertex_accessor.property(
+            "country", args[2]
+        );
+        vertex_accessor.property(
+            "created_at", args[3]
+        );
+        auto label = db.graph.label_store.find_or_create("ACCOUNT");
+        vertex_accessor.add_label(label);
+        t.commit();
+        // TODO: return the result once the bolt will be implemented
+    };
+
+    auto find_node_by_id = [&db](const properties_t& args)
+    {
+        auto& t = db.tx_engine.begin();
+        auto id = static_cast<Int64&>(*args[0]);
+        auto vertex_accessor = db.graph.vertices.find(t, Id(id.value));
+        t.commit();
+        // TODO: return the result once the bolt will be implemented
+    };
+
+    queries[10597108978382323595u] = create_account;
+
+    auto arguments = all_arguments(argc, argv);
+    auto input_query = extract_query(arguments);
+
+    auto stripper = make_query_stripper(TK_INT, TK_FLOAT, TK_STR, TK_BOOL);
+    auto stripped = stripper.strip(input_query);
+    
+    auto time = timer<ms>([&stripped, &queries]() {
+        for (int i = 0; i < 1000000; ++i) {
+            queries[stripped.hash](stripped.arguments);
+        }
+    });
+    std::cout << time << std::endl;
+
+    return 0;
+}

src/query_engine/main_query_hasher.cpp

@@ -0,0 +1,38 @@
+#include <iostream>
+
+#include "query_hasher.hpp"
+#include "cypher/common.hpp"
+#include "query_stripper.hpp"
+#include "utils/command_line/arguments.hpp"
+#include "utils/type_discovery.hpp"
+
+using std::cout;
+using std::cin;
+using std::endl;
+
+int main(int argc, char** argv)
+{
+    // arguments parsing
+    auto arguments = all_arguments(argc, argv);
+
+    // query extraction
+    auto input_query = extract_query(arguments);
+
+    cout << "QUERY: " << input_query << endl;
+    
+    auto stripper = make_query_stripper(TK_INT, TK_FLOAT, TK_STR, TK_BOOL);
+    auto stripped = stripper.strip(input_query);
+
+    cout << "STRIPPED QUERY: " << stripped.query << endl;
+
+    QueryHasher query_hasher;
+
+    cout << "QUERY HASH: " << query_hasher.hash(stripped.query) << endl;
+    
+    cout << "PROPERTIES:" << endl;
+    for (auto property : stripped.arguments) {
+        cout << "    " << *property << endl;
+    }
+
+    return 0;
+}

src/query_engine/program_loader.hpp

@@ -34,8 +34,7 @@ public:
         auto stripped = stripper.strip(query);
         LOG_INFO("stripped_query=" + stripped.query);
 
-        auto stripped_hash = fnv(stripped.query);
-        auto hash_string = std::to_string(stripped_hash);
+        auto hash_string = std::to_string(stripped.hash);
         LOG_INFO("query_hash=" + hash_string);
 
         auto code_lib_iter = code_libs.find(stripped_hash);

src/query_engine/query_hasher.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "cypher/common.hpp"
+#include "query_stripper.hpp"
+#include "utils/hashing/fnv.hpp"
+
+class QueryHasher
+{
+public:
+    QueryHasher()
+        : stripper(make_query_stripper(TK_INT, TK_FLOAT, TK_STR, TK_BOOL)) {}
+
+    std::string hash(std::string &query)
+    {
+        auto stripped = stripper.strip(query);
+        auto stripped_hash = fnv(stripped.query);
+        auto hash_string = std::to_string(stripped_hash);
+        return hash_string;
+    }
+    
+private:
+    QueryStripper<int, int, int, int> stripper;
+};

src/query_engine/query_stripped.hpp

@@ -8,8 +8,9 @@ using code_args_t = std::vector<Property::sptr>;
 
 struct QueryStripped
 {
-    QueryStripped(const std::string &&query, code_args_t &&arguments)
-        : query(std::forward<const std::string>(query)),
+    QueryStripped(const std::string &&query, uint64_t hash,
+                  code_args_t &&arguments)
+        : query(std::forward<const std::string>(query)), hash(hash),
           arguments(std::forward<code_args_t>(arguments))
     {
     }
@@ -17,6 +18,7 @@ struct QueryStripped
     QueryStripped(QueryStripped &other) = delete;
     QueryStripped(QueryStripped &&other) = default;
 
-    std::string query;
     code_args_t arguments;
+    uint64_t hash;
+    std::string query;
 };

src/query_engine/query_stripper.hpp

@@ -7,6 +7,7 @@
 
 #include "cypher/cypher.h"
 #include "cypher/tokenizer/cypher_lexer.hpp"
+#include "utils/hashing/fnv.hpp"
 #include "query_stripped.hpp"
 #include "storage/model/properties/all.hpp"
 #include "utils/string/transform.hpp"
@@ -84,6 +85,7 @@ public:
         }
 
         return QueryStripped(std::move(stripped_query),
+                             fnv(stripped_query),
                              std::move(stripped_arguments));
     }
 

src/storage/graph.hpp

@@ -1,14 +1,15 @@
 #pragma once
 
-#include "data_structures/skiplist/skiplist.hpp"
 #include "storage/vertices.hpp"
 #include "storage/edges.hpp"
+#include "storage/label_store.hpp"
 
 class Graph
 {
 public:
     Graph() {}
 
-    Vertices vertices;
     Edges edges;
+    Vertices vertices;
+    LabelStore label_store;
 };

src/storage/label_store.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <stdexcept>
+
 #include "model/label.hpp"
 #include "data_structures/skiplist/skiplistset.hpp"
 
@@ -13,12 +15,15 @@ public:
         return accessor.insert(Label(name)).first;
     }
 
-    bool contains(const std::string& name) const
+    bool contains(const std::string& name) // const
     {
         auto accessor = labels.access();
         return accessor.find(Label(name)) != accessor.end();
     }
 
+    // TODO: implement find method
+    //       return { Label, is_found }
+
 private:
     SkipListSet<Label> labels;
 };

src/storage/model/label_collection.hpp

@@ -3,7 +3,7 @@
 #include <set>
 #include "label.hpp"
 
-class LabelList
+class LabelCollection
 {
 public:
     auto begin() { return labels.begin(); }
@@ -16,7 +16,7 @@ public:
 
     bool add(const Label& label)
     {
-        return labels.insert(label).second;
+        return labels.insert(std::cref(label)).second;
     }
 
     bool has(const Label& label) const
@@ -44,5 +44,5 @@ public:
     }
 
 private:
-    std::set<const Label&> labels;
+    std::set<std::reference_wrapper<const Label>> labels;
 };

src/storage/model/properties/property.hpp

@@ -4,6 +4,7 @@
 #include <string>
 #include <cassert>
 #include <ostream>
+#include <vector>
 
 #include "utils/underlying_cast.hpp"
 
@@ -102,3 +103,4 @@ public:
     const Flags flags;
 };
 
+using properties_t = std::vector<Property::sptr>;

src/storage/model/vertex_model.hpp

@@ -1,12 +1,12 @@
 #pragma once
 
 #include "property_model.hpp"
-// #include "label_list.hpp"
+#include "storage/model/label_collection.hpp"
 #include "edge_list.hpp"
 
 class VertexModel : public PropertyModel
 {
 public:
     EdgeList in, out;
-    // LabelList labels;
+    LabelCollection labels;
 };

src/storage/vertex_accessor.hpp

@@ -20,8 +20,18 @@ public:
         return this->record->data.in.degree();
     }
     
-    size_t degree()
+    size_t degree() const
     {
         return in_degree() + out_degree();
     }
+
+    bool add_label(const Label& label)
+    {
+        return this->record->data.labels.add(label);
+    }
+
+    bool has_label(const Label& label) const
+    {
+        return this->record->data.labels.has(label);
+    }
 };

src/template_engine/engine.hpp

@@ -17,8 +17,7 @@ public:
     string render(const string& form, const data& partials)
     {
         //  TODO more optimal implementation
-        //  if more optimal implementation is too expensive
-        //  use some templating engine like https://github.com/no1msd/mstch
+        //  another option is something like https://github.com/no1msd/mstch
         //  but it has to be wrapped
         string rendered = form;
         for (auto partial : partials) {

src/threading/sync/spinlock.hpp

@@ -23,5 +23,5 @@ public:
 
 private:
     // guaranteed by standard to be lock free!
-    std::atomic_flag lock_flag = ATOMIC_FLAG_INIT;
+    mutable std::atomic_flag lock_flag = ATOMIC_FLAG_INIT;
 };

src/utils/time/timer.hpp

@@ -6,6 +6,9 @@
 
 #define time_now() std::chrono::high_resolution_clock::now()
 
+using ns = std::chrono::nanoseconds;
+using ms = std::chrono::milliseconds;
+
 template <typename DurationUnit = std::chrono::nanoseconds>
 auto to_duration(const std::chrono::duration<long, std::nano> &delta)
 {

tests/CMakeLists.txt

@@ -16,17 +16,21 @@ message(STATUS "Available unit tests are: ${unit_test_names}")
 file(COPY ${CMAKE_SOURCE_DIR}/tests/data
      DESTINATION ${CMAKE_BINARY_DIR}/tests)
 
+set(chosen_test "concurrent_skiplist")
+
 # build unit tests
 foreach(test ${unit_test_names})
     set(test_name unit_${test})
-    add_executable(${test_name} unit/${test}.cpp)
-    # TODO: separate dependencies
-    target_link_libraries(${test_name} stdc++fs)
-    target_link_libraries(${test_name} cypher_lib)
-    target_link_libraries(${test_name} Threads::Threads)
-    target_link_libraries(${test_name} ${fmt_static_lib})
-    add_test(NAME ${test_name} COMMAND ${test_name})
-    set_property(TARGET ${test_name} PROPERTY CXX_STANDARD 14)
+    if (${chosen_test} STREQUAL ${test_name})
+        add_executable(${test_name} unit/${test}.cpp)
+        # TODO: separate dependencies
+        target_link_libraries(${test_name} stdc++fs)
+        target_link_libraries(${test_name} cypher_lib)
+        target_link_libraries(${test_name} Threads::Threads)
+        target_link_libraries(${test_name} ${fmt_static_lib})
+        add_test(NAME ${test_name} COMMAND ${test_name})
+        set_property(TARGET ${test_name} PROPERTY CXX_STANDARD 14)
+    endif(${chosen_test} STREQUAL ${test_name})
 endforeach()
 
 ## CONCURRENCY TESTS
@@ -41,8 +45,10 @@ message(STATUS "Available concurrency tests are: ${concurrency_test_names}")
 # build concurrency tests
 foreach(test ${concurrency_test_names})
     set(test_name concurrent_${test})
-    add_executable(${test_name} concurrent/${test}.cpp)
-    target_link_libraries(${test_name} Threads::Threads)
-    add_test(NAME ${test_name} COMMAND ${test_name})
-    set_property(TARGET ${test_name} PROPERTY CXX_STANDARD 14)
+    if (${chosen_test} STREQUAL ${test_name})
+        add_executable(${test_name} concurrent/${test}.cpp)
+        target_link_libraries(${test_name} Threads::Threads)
+        add_test(NAME ${test_name} COMMAND ${test_name})
+        set_property(TARGET ${test_name} PROPERTY CXX_STANDARD 14)
+    endif(${chosen_test} STREQUAL ${test_name})
 endforeach()

tests/concurrent/skiplist.cpp

@@ -12,16 +12,14 @@ using std::endl;
 using skiplist_t = SkipList<int, int>;
 using namespace std::chrono_literals;
 
-#define THREADS_NO 16
-constexpr size_t elems_per_thread = 100000;
+#define THREADS_NO 1
+constexpr size_t elems_per_thread = 1000;
 
 int main()
 {
     ds::static_array<std::thread, THREADS_NO> threads;
     skiplist_t skiplist;
     
-    cout << "1. used virtual memory: " << used_virtual_memory() << endl;
-
     // put THREADS_NO * elems_per_thread items to the skiplist
     for (size_t thread_i = 0; thread_i < THREADS_NO; ++thread_i) {
         threads[thread_i] = std::thread(
@@ -34,14 +32,11 @@ int main()
             thread_i * elems_per_thread,
             thread_i * elems_per_thread + elems_per_thread);
     }
-
     // wait all threads
     for (auto &thread : threads) {
         thread.join();
     }
 
-    cout << "1. used virtual memory: " << used_virtual_memory() << endl;
-
     // get skiplist size
     {
         auto accessor = skiplist.access();
@@ -60,14 +55,11 @@ int main()
             thread_i * elems_per_thread,
             thread_i * elems_per_thread + elems_per_thread);
     }
-
     // wait all threads
     for (auto &thread : threads) {
         thread.join();
     }
 
-    cout << "1. used virtual memory: " << used_virtual_memory() << endl;
-
     // check size
     {
         auto accessor = skiplist.access();
@@ -85,9 +77,9 @@ int main()
         permanent_assert(iterator_counter == 0, "deleted elements");
     }
 
+    std::this_thread::sleep_for(100s);
+
     // TODO: test GC and memory
-    std::this_thread::sleep_for(5s);
-    cout << "1. used virtual memory: " << used_virtual_memory() << endl;
 
     return 0;
 }

tests/unit/skiplistset.cpp

@@ -0,0 +1,54 @@
+#include <iostream>
+
+#include "data_structures/skiplist/skiplistset.hpp"
+
+using std::cout;
+using std::endl;
+
+void print_skiplist(const SkipListSet<int>::Accessor& skiplist)
+{
+    cout << "---- skiplist set now has: ";
+
+    for(auto& item : skiplist)
+        cout << item << ", ";
+
+    cout << "----" << endl;
+}
+
+int main(void)
+{
+    SkipListSet<int> set;
+    auto accessor = set.access();
+
+    cout << std::boolalpha;
+
+    cout << "added non-existing 1? (true) "
+         << accessor.insert(1).second << endl;
+
+    cout << "added already existing 1? (false) "
+         << accessor.insert(1).second << endl;
+
+    accessor.insert(2);
+    print_skiplist(accessor);
+
+    cout << "item 3 doesn't exist? (true) "
+         << (accessor.find(3) == accessor.end()) << endl;
+
+    cout << "item 3 exists? (false) "
+         << accessor.contains(3) << endl;
+
+    cout << "item 2 exists? (true) "
+         << (accessor.find(2) != accessor.end()) << endl;
+
+    cout << "at item 2 is? 2 " << *accessor.find(2) << endl;
+
+    cout << "removed existing 1? (true) " << accessor.remove(1) << endl;
+    cout << "removed non-existing 3? (false) " << accessor.remove(3) << endl;
+
+    accessor.insert(1);
+    accessor.insert(4);
+
+    print_skiplist(accessor);
+
+    return 0;
+}