First integration almost done. BOLT still doesn't work as expected, problem is only serialization order.

This commit is contained in:
Marko Budiselic 2016-08-11 04:47:30 +01:00
parent 35d8f6d7ab
commit 93b174afd7
94 changed files with 430 additions and 285 deletions
CMakeLists.txt
include
src
tests

View File

@ -231,6 +231,7 @@ FILE(COPY ${include_dir}/storage/indexes/index_record.hpp DESTINATION ${build_in
FILE(COPY ${include_dir}/storage/indexes/index_record_collection.hpp DESTINATION ${build_include_dir}/storage/indexes)
FILE(COPY ${include_dir}/utils/sys.hpp DESTINATION ${build_include_dir}/utils)
FILE(COPY ${include_dir}/utils/bswap.hpp DESTINATION ${build_include_dir}/utils)
FILE(COPY ${include_dir}/utils/stacktrace.hpp DESTINATION ${build_include_dir}/utils)
FILE(COPY ${include_dir}/utils/auto_scope.hpp DESTINATION ${build_include_dir}/utils)
FILE(COPY ${include_dir}/utils/assert.hpp DESTINATION ${build_include_dir}/utils)
@ -246,6 +247,31 @@ FILE(COPY ${include_dir}/utils/counters/simple_counter.hpp DESTINATION ${build_i
FILE(COPY ${include_dir}/utils/random/fast_binomial.hpp DESTINATION ${build_include_dir}/utils/random)
FILE(COPY ${include_dir}/utils/random/xorshift128plus.hpp DESTINATION ${build_include_dir}/utils/random)
FILE(COPY ${include_dir}/utils/exceptions/basic_exception.hpp DESTINATION ${build_include_dir}/utils/exceptions)
FILE(COPY ${include_dir}/utils/datetime/timestamp.hpp DESTINATION ${build_include_dir}/utils/datetime)
FILE(COPY ${include_dir}/utils/datetime/datetime_error.hpp DESTINATION ${build_include_dir}/utils/datetime)
FILE(COPY ${include_dir}/utils/types/byte.hpp DESTINATION ${build_include_dir}/utils/types)
FILE(COPY ${include_dir}/communication/communication.hpp DESTINATION ${build_include_dir}/communication)
FILE(COPY ${include_dir}/communication/bolt/v1/config.hpp DESTINATION ${build_include_dir}/communication/bolt/v1)
FILE(COPY ${include_dir}/communication/bolt/v1/serialization/record_stream.hpp DESTINATION ${build_include_dir}/communication/bolt/v1/serialization)
FILE(COPY ${include_dir}/communication/bolt/v1/serialization/bolt_serializer.hpp DESTINATION ${build_include_dir}/communication/bolt/v1/serialization)
FILE(COPY ${include_dir}/communication/bolt/v1/transport/bolt_encoder.hpp DESTINATION ${build_include_dir}/communication/bolt/v1/transport)
FILE(COPY ${include_dir}/communication/bolt/v1/transport/chunked_buffer.hpp DESTINATION ${build_include_dir}/communication/bolt/v1/transport)
FILE(COPY ${include_dir}/communication/bolt/v1/transport/chunked_encoder.hpp DESTINATION ${build_include_dir}/communication/bolt/v1/transport)
FILE(COPY ${include_dir}/communication/bolt/v1/transport/socket_stream.hpp DESTINATION ${build_include_dir}/communication/bolt/v1/transport)
FILE(COPY ${include_dir}/communication/bolt/v1/transport/stream_error.hpp DESTINATION ${build_include_dir}/communication/bolt/v1/transport)
FILE(COPY ${include_dir}/communication/bolt/v1/packing/codes.hpp DESTINATION ${build_include_dir}/communication/bolt/v1/packing)
FILE(COPY ${include_dir}/communication/bolt/v1/messaging/codes.hpp DESTINATION ${build_include_dir}/communication/bolt/v1/messaging)
FILE(COPY ${include_dir}/io/network/socket.hpp DESTINATION ${build_include_dir}/io/network)
FILE(COPY ${include_dir}/io/network/addrinfo.hpp DESTINATION ${build_include_dir}/io/network)
FILE(COPY ${include_dir}/io/network/network_error.hpp DESTINATION ${build_include_dir}/io/network)
FILE(COPY ${include_dir}/io/network/socket.hpp DESTINATION ${build_include_dir}/io/network)
FILE(COPY ${include_dir}/logging/default.hpp DESTINATION ${build_include_dir}/logging)
FILE(COPY ${include_dir}/logging/log.hpp DESTINATION ${build_include_dir}/logging)
FILE(COPY ${include_dir}/logging/logger.hpp DESTINATION ${build_include_dir}/logging)
FILE(COPY ${include_dir}/logging/levels.hpp DESTINATION ${build_include_dir}/logging)
# -----------------------------------------------------------------------------
# add all cpp file recursive into sourceFiles varibale
@ -264,25 +290,67 @@ endif()
set(CMAKE_CXX_FLAGS_RELEASE
"${CMAKE_CXX_FLAGS_RELEASE} -march=native -Wall -Werror")
# -- configure time variables -------------------------------------------------
option(RUNTIME_ASSERT "Enable runtime assertions" OFF)
# -- configure defines -- default is ON | true | enabled ----------------------
# -- logging ------------------------------------------------------------------
option(LOG_NO_TRACE "Disable trace logging" OFF)
message(STATUS "LOG_NO_TRACE: ${LOG_NO_TRACE}")
if (LOG_NO_TRACE)
add_definitions(-DLOG_NO_TRACE)
endif()
option(LOG_NO_DEBUG "Disable debug logging" OFF)
message(STATUS "LOG_NO_DEBUG: ${LOG_NO_DEBUG}")
if (LOG_NO_DEBUG)
add_definitions(-DLOG_NO_DEBUG)
endif()
option(LOG_NO_INFO "Disable info logging" OFF)
message(STATUS "LOG_NO_INFO: ${LOG_NO_INFO}")
if (LOG_NO_INFO)
add_definitions(-DLOG_NO_INFO)
endif()
option(LOG_NO_WARN "Disable warn logging" OFF)
message(STATUS "LOG_NO_WARN: ${LOG_NO_WARN}")
if (LOG_NO_WARN)
add_definitions(-DLOG_NO_WARN)
endif()
option(LOG_NO_ERROR "Disable error logging" OFF)
message(STATUS "LOG_NO_ERROR: ${LOG_NO_ERROR}")
if (LOG_NO_ERROR)
add_definitions(-DLOG_NO_ERROR)
endif()
# -- logging ------------------------------------------------------------------
# -- assert -------------------------------------------------------------------
option(RUNTIME_ASSERT "Enable runtime assertions" ON)
message(STATUS "RUNTIME_ASSERT: ${RUNTIME_ASSERT}")
if(RUNTIME_ASSERT)
add_definitions( -DRUNTIME_ASSERT_ON )
add_definitions(-DRUNTIME_ASSERT_ON)
endif()
option(THROW_EXCEPTION_ON_ERROR "Throw exception on error" OFF)
option(THROW_EXCEPTION_ON_ERROR "Throw exception on error" ON)
message(STATUS "THROW_EXCEPTION_ON_ERROR: ${THROW_EXCEPTION_ON_ERROR}")
if(THROW_EXCEPTION_ON_ERROR)
add_definitions( -DTHROW_EXCEPTION_ON_ERROR )
add_definitions(-DTHROW_EXCEPTION_ON_ERROR)
endif()
# -- assert -------------------------------------------------------------------
# -- ndebug -------------------------------------------------------------------
option(NDEBUG "No debug" OFF)
message(STATUS "NDEBUG: ${NDEBUG} (be careful CMAKE_BUILD_TYPE can also append this flag)")
if(NDEBUG)
add_definitions( -DNDEBUG )
endif()
# -- ndebug -------------------------------------------------------------------
# -- binaries -----------------------------------------------------------------
option(MEMGRAPH "Build memgraph binary" ON)
message(STATUS "MEMGRAPH binary: ${MEMGRAPH}")
option(POC "Build proof of concept binaries" ON)
message(STATUS "POC binaries: ${POC}")
option(TESTS "Build test binaries" ON)
message(STATUS "TESTS binaries: ${TESTS}")
# -- binaries -----------------------------------------------------------------
# -- configure defines --------------------------------------------------------
# -- includes -----------------------------------------------------------------
include_directories(${CMAKE_SOURCE_DIR}/include)
@ -311,15 +379,15 @@ set(memgraph_src_files
${src_dir}/utils/string/join.cpp
${src_dir}/utils/string/file.cpp
${src_dir}/query_engine/util.cpp
${src_dir}/bolt/v1/bolt.cpp
${src_dir}/bolt/v1/states.cpp
${src_dir}/bolt/v1/session.cpp
${src_dir}/bolt/v1/states/error.cpp
${src_dir}/bolt/v1/states/executor.cpp
${src_dir}/bolt/v1/states/init.cpp
${src_dir}/bolt/v1/states/handshake.cpp
${src_dir}/bolt/v1/transport/bolt_decoder.cpp
${src_dir}/bolt/v1/transport/buffer.cpp
${src_dir}/communication/bolt/v1/bolt.cpp
${src_dir}/communication/bolt/v1/states.cpp
${src_dir}/communication/bolt/v1/session.cpp
${src_dir}/communication/bolt/v1/states/error.cpp
${src_dir}/communication/bolt/v1/states/executor.cpp
${src_dir}/communication/bolt/v1/states/init.cpp
${src_dir}/communication/bolt/v1/states/handshake.cpp
${src_dir}/communication/bolt/v1/transport/bolt_decoder.cpp
${src_dir}/communication/bolt/v1/transport/buffer.cpp
${src_dir}/mvcc/id.cpp
${src_dir}/storage/vertices.cpp
${src_dir}/storage/edges.cpp

View File

@ -1,6 +1,6 @@
#pragma once
#include "bolt/v1/states.hpp"
#include "communication/bolt/v1/states.hpp"
#include "io/network/socket.hpp"
#include "dbms/dbms.hpp"

View File

@ -1,7 +1,7 @@
#pragma once
#include "bolt/v1/transport/bolt_encoder.hpp"
#include "bolt/v1/packing/codes.hpp"
#include "communication/bolt/v1/transport/bolt_encoder.hpp"
#include "communication/bolt/v1/packing/codes.hpp"
#include "storage/vertex_accessor.hpp"
#include "storage/edge_accessor.hpp"
@ -17,6 +17,10 @@ class BoltSerializer
{
friend class Property;
// TODO: here shoud be friend but it doesn't work
// template <class Handler>
// friend void accept(const Property &property, Handler &h);
public:
BoltSerializer(Stream& stream) : encoder(stream) {}
@ -51,8 +55,10 @@ public:
encoder.write_map_header(props.size());
for(auto& prop : props)
write(prop);
for(auto& prop : props) {
write(prop.first);
write(*prop.second);
}
}
/* Serializes the vertex accessor into the packstream format
@ -87,8 +93,10 @@ public:
encoder.write_map_header(props.size());
for(auto& prop : props)
write(prop);
for(auto& prop : props) {
write(prop.first);
write(*prop.second);
}
}
void write(const Property& prop)
@ -126,19 +134,24 @@ public:
encoder.write_integer(prop.value);
}
void write(const std::string& value)
{
encoder.write_string(value);
}
void write(const String& prop)
{
encoder.write_string(prop.value);
}
protected:
Stream& encoder;
template <class T>
void handle(const T& prop)
{
write(prop);
}
protected:
Stream& encoder;
};
}

View File

@ -1,9 +1,9 @@
#pragma once
#include "bolt/v1/serialization/bolt_serializer.hpp"
#include "bolt/v1/transport/chunked_buffer.hpp"
#include "bolt/v1/transport/chunked_encoder.hpp"
#include "bolt/v1/transport/socket_stream.hpp"
#include "communication/bolt/v1/serialization/bolt_serializer.hpp"
#include "communication/bolt/v1/transport/chunked_buffer.hpp"
#include "communication/bolt/v1/transport/chunked_encoder.hpp"
#include "communication/bolt/v1/transport/socket_stream.hpp"
#include "logging/default.hpp"
@ -57,6 +57,16 @@ public:
flush();
}
void write_field(const std::string& field)
{
bolt_encoder.message_success();
bolt_encoder.write_map_header(1);
bolt_encoder.write_string("fields");
write_list_header(1);
bolt_encoder.write_string(field);
flush();
}
void write_list_header(size_t size)
{
bolt_encoder.write_list_header(size);

View File

@ -7,7 +7,7 @@
#include <cassert>
#include "io/network/server.hpp"
#include "bolt/v1/bolt.hpp"
#include "communication/bolt/v1/bolt.hpp"
namespace bolt
{

View File

@ -1,19 +1,16 @@
#pragma once
#include <iomanip>
#include <cstdio>
#include <atomic>
#include <sstream>
#include <cstdio>
#include <iomanip>
#include <memory>
#include <sstream>
#include <thread>
#include "io/network/stream_reader.hpp"
#include "bolt/v1/bolt.hpp"
#include "bolt/v1/session.hpp"
#include "communication/bolt/v1/bolt.hpp"
#include "communication/bolt/v1/session.hpp"
#include "logging/default.hpp"
#include "io/network/stream_reader.hpp"
namespace bolt
{
@ -28,19 +25,19 @@ class Worker : public io::StreamReader<Worker, Session>
public:
using sptr = std::shared_ptr<Worker>;
Worker(Bolt& bolt) : bolt(bolt)
Worker(Bolt &bolt) : bolt(bolt)
{
logger = logging::log->logger("Network");
}
Session& on_connect(io::Socket&& socket)
Session &on_connect(io::Socket &&socket)
{
logger.trace("Accepting connection on socket {}", socket.id());
return *bolt.get().create_session(std::forward<io::Socket>(socket));
}
void on_error(Session&)
void on_error(Session &)
{
logger.trace("[on_error] errno = {}", errno);
@ -50,43 +47,39 @@ public:
#endif
logger.error("Error occured in this session");
}
void on_wait_timeout() {}
Buffer on_alloc(Session&)
Buffer on_alloc(Session &)
{
/* logger.trace("[on_alloc] Allocating {}B", sizeof buf); */
return Buffer { buf, sizeof buf };
return Buffer{buf, sizeof buf};
}
void on_read(Session& session, Buffer& buf)
void on_read(Session &session, Buffer &buf)
{
logger.trace("[on_read] Received {}B", buf.len);
#ifndef NDEBUG
std::stringstream stream;
for(size_t i = 0; i < buf.len; ++i)
for (size_t i = 0; i < buf.len; ++i)
stream << fmt::format("{:02X} ", static_cast<byte>(buf.ptr[i]));
logger.trace("[on_read] {}", stream.str());
#endif
try
{
session.execute(reinterpret_cast<const byte*>(buf.ptr), buf.len);
}
catch(const std::exception& e)
{
try {
session.execute(reinterpret_cast<const byte *>(buf.ptr), buf.len);
} catch (const std::exception &e) {
logger.error("Error occured while executing statement.");
logger.error("{}", e.what());
}
}
void on_close(Session& session)
void on_close(Session &session)
{
logger.trace("[on_close] Client closed the connection");
session.close();
@ -100,13 +93,12 @@ protected:
Logger logger;
std::thread thread;
void start(std::atomic<bool>& alive)
void start(std::atomic<bool> &alive)
{
thread = std::thread([&, this]() {
while(alive)
while (alive)
wait_and_process_events();
});
}
};
}

View File

@ -3,11 +3,12 @@
#include "io/network/socket.hpp"
#include "io/network/tcp/stream.hpp"
#include "bolt/v1/bolt.hpp"
#include "bolt/v1/serialization/record_stream.hpp"
#include "bolt/v1/states/state.hpp"
#include "bolt/v1/transport/bolt_decoder.hpp"
#include "bolt/v1/transport/bolt_encoder.hpp"
#include "communication/bolt/v1/bolt.hpp"
#include "communication/bolt/v1/serialization/record_stream.hpp"
#include "communication/bolt/v1/states/state.hpp"
#include "communication/bolt/v1/transport/bolt_decoder.hpp"
#include "communication/bolt/v1/transport/bolt_encoder.hpp"
#include "communication/communication.hpp"
#include "logging/default.hpp"
@ -18,7 +19,7 @@ class Session : public io::tcp::Stream<io::Socket>
{
public:
using Decoder = BoltDecoder;
using OutputStream = RecordStream<io::Socket>;
using OutputStream = communication::OutputStream;
Session(io::Socket &&socket, Bolt &bolt);

View File

@ -1,6 +1,6 @@
#pragma once
#include "bolt/v1/states/state.hpp"
#include "communication/bolt/v1/states/state.hpp"
#include "logging/log.hpp"
namespace bolt

View File

@ -1,7 +1,7 @@
#pragma once
#include "bolt/v1/session.hpp"
#include "bolt/v1/states/state.hpp"
#include "communication/bolt/v1/session.hpp"
#include "communication/bolt/v1/states/state.hpp"
namespace bolt
{

View File

@ -1,7 +1,7 @@
#pragma once
#include "bolt/v1/states/state.hpp"
#include "bolt/v1/session.hpp"
#include "communication/bolt/v1/states/state.hpp"
#include "communication/bolt/v1/session.hpp"
#include "query_engine/query_engine.hpp"
namespace bolt

View File

@ -1,6 +1,6 @@
#pragma once
#include "bolt/v1/states/state.hpp"
#include "communication/bolt/v1/states/state.hpp"
namespace bolt
{

View File

@ -1,6 +1,6 @@
#pragma once
#include "bolt/v1/states/message_parser.hpp"
#include "communication/bolt/v1/states/message_parser.hpp"
namespace bolt
{

View File

@ -1,7 +1,7 @@
#pragma once
#include "bolt/v1/session.hpp"
#include "bolt/v1/states/state.hpp"
#include "communication/bolt/v1/session.hpp"
#include "communication/bolt/v1/states/state.hpp"
#include "utils/crtp.hpp"
namespace bolt

View File

@ -1,7 +1,7 @@
#pragma once
#include "bolt/v1/transport/buffer.hpp"
#include "bolt/v1/transport/chunked_decoder.hpp"
#include "communication/bolt/v1/transport/buffer.hpp"
#include "communication/bolt/v1/transport/chunked_decoder.hpp"
#include "utils/types/byte.hpp"
namespace bolt

View File

@ -2,8 +2,8 @@
#include <string>
#include "bolt/v1/packing/codes.hpp"
#include "bolt/v1/messaging/codes.hpp"
#include "communication/bolt/v1/packing/codes.hpp"
#include "communication/bolt/v1/messaging/codes.hpp"
#include "utils/types/byte.hpp"
#include "utils/bswap.hpp"
#include "logging/default.hpp"

View File

@ -4,7 +4,7 @@
#include <vector>
#include <cstring>
#include "bolt/v1/config.hpp"
#include "communication/bolt/v1/config.hpp"
#include "utils/types/byte.hpp"
#include "logging/default.hpp"

View File

@ -5,7 +5,7 @@
#include <functional>
#include "utils/likely.hpp"
#include "bolt/v1/config.hpp"
#include "communication/bolt/v1/config.hpp"
#include "logging/default.hpp"
namespace bolt

View File

@ -5,7 +5,7 @@
#include <cstdio>
#include "io/network/socket.hpp"
#include "bolt/v1/transport/stream_error.hpp"
#include "communication/bolt/v1/transport/stream_error.hpp"
namespace bolt
{

View File

@ -0,0 +1,9 @@
#pragma once
#include "io/network/socket.hpp"
#include "communication/bolt/v1/serialization/record_stream.hpp"
namespace communication
{
using OutputStream = bolt::RecordStream<io::Socket>;
}

View File

@ -0,0 +1,4 @@
#pragma once
/* Memgraph Communication protocol
* gate is the first name proposal for the protocol */

View File

@ -0,0 +1,3 @@
#pragma once
/* HTTP & HTTPS implementation */

View File

@ -6,11 +6,14 @@
#include "memory/freelist.hpp"
#include "memory/lazy_gc.hpp"
#include "threading/sync/spinlock.hpp"
#include "logging/default.hpp"
template <class T, class lock_t = SpinLock>
class SkiplistGC : public LazyGC<SkiplistGC<T, lock_t>, lock_t>
{
public:
SkiplistGC() : logger(logging::log->logger("SkiplistGC")) {}
// release_ref method should be called by a thread
// when the thread finish it job over object
// which has to be lazy cleaned
@ -33,9 +36,8 @@ public:
}
if (local_freelist.size() > 0) {
std::cout << "GC started" << std::endl;
std::cout << "Local list size: " << local_freelist.size()
<< std::endl;
logger.trace("GC started");
logger.trace("Local list size: {}", local_freelist.size());
long long counter = 0;
// destroy all elements from local_freelist
for (auto element : local_freelist) {
@ -45,13 +47,15 @@ public:
counter++;
}
}
std::cout << "Number of destroyed elements " << counter
<< std::endl;
logger.trace("Number of destroyed elements: {}", counter);
}
}
void collect(T *node) { freelist.add(node); }
protected:
Logger logger;
private:
FreeList<T> freelist;
};

View File

@ -3,7 +3,7 @@
#include <cstring>
#include <netdb.h>
#include "network_error.hpp"
#include "io/network/network_error.hpp"
#include "utils/underlying_cast.hpp"
namespace io

View File

@ -1,6 +1,6 @@
#pragma once
#include "stream_reader.hpp"
#include "io/network/stream_reader.hpp"
namespace io
{

View File

@ -3,7 +3,7 @@
#include <malloc.h>
#include <sys/epoll.h>
#include "socket.hpp"
#include "io/network/socket.hpp"
#include "utils/likely.hpp"
namespace io

View File

@ -1,6 +1,6 @@
#pragma once
#include "epoll.hpp"
#include "io/network/epoll.hpp"
#include "utils/crtp.hpp"
namespace io

View File

@ -1,6 +1,6 @@
#pragma once
#include "socket.hpp"
#include "io/network/socket.hpp"
namespace io
{

View File

@ -2,7 +2,7 @@
#include <openssl/ssl.h>
#include "stream_reader.hpp"
#include "io/network/stream_reader.hpp"
#include "logging/default.hpp"
namespace io

View File

@ -1,6 +1,6 @@
#pragma once
#include "stream_reader.hpp"
#include "io/network/stream_reader.hpp"
namespace io
{

View File

@ -13,7 +13,7 @@
#include <sys/epoll.h>
#include <errno.h>
#include "addrinfo.hpp"
#include "io/network/addrinfo.hpp"
#include "utils/likely.hpp"
#include "logging/default.hpp"
@ -48,8 +48,9 @@ public:
if(socket == -1)
return;
std::cout << "DELETING SOCKET" << std::endl;
#ifndef NDEBUG
logging::debug("DELETING SOCKET");
#endif
::close(socket);
}
@ -169,6 +170,7 @@ public:
int write(const byte* data, size_t len)
{
// TODO: use logger
#ifndef NDEBUG
std::stringstream stream;
@ -189,6 +191,7 @@ public:
}
protected:
Logger logger;
int socket;
};

View File

@ -1,6 +1,5 @@
#pragma once
#include "
namespace io
{

View File

@ -1,6 +1,6 @@
#pragma once
#include "event_listener.hpp"
#include "io/network/event_listener.hpp"
namespace io
{

View File

@ -1,6 +1,6 @@
#pragma once
#include "stream_listener.hpp"
#include "io/network/stream_listener.hpp"
#include "memory/literals.hpp"
namespace io

View File

@ -16,6 +16,14 @@ void debug(Args&&... args)
debug_logger.debug(std::forward<Args>(args)...);
}
extern Logger info_logger;
template <class... Args>
void info(Args&&... args)
{
info_logger.info(std::forward<Args>(args)...);
}
void init_async();
void init_sync();

View File

@ -3,30 +3,32 @@
#include <string>
#include "exceptions/exceptions.hpp"
#include "logging/default.hpp"
#include "utils/string/join.hpp"
// TODO:
// * all libraries have to be compiled in the server compile time
// * compile command has to be generated
#include <iostream>
class CodeCompiler
{
public:
CodeCompiler() : logger(logging::log->logger("CodeCompiler")) {}
void compile(const std::string &in_file, const std::string &out_file)
{
// generate compile command
auto compile_command =
utils::prints("clang++",
// "-std=c++1y -O2 -DNDEBUG", // compile flags
"-std=c++1y -DDEBUG", // compile flags // TODO: load from config file
in_file, // input file
"-o", out_file, // ouput file
"-I./include", // include paths (TODO: parameter)
"-I../libs/fmt", // TODO: load from config
"-L./ -lmemgraph_pic",
"-shared -fPIC" // shared library flags
);
auto compile_command = utils::prints(
"clang++",
// "-std=c++1y -O2 -DNDEBUG", // compile flags
"-std=c++1y", // compile flags // TODO: load from config file
in_file, // input file
"-o", out_file, // ouput file
"-I./include", // include paths (TODO: parameter)
"-I../libs/fmt", // TODO: load from config
"-L./ -lmemgraph_pic",
"-shared -fPIC" // shared library flags
);
// synchronous call
auto compile_status = system(compile_command.c_str());
@ -38,9 +40,10 @@ public:
"settings are wrong");
}
// TODO: use logger
std::cout << fmt::format("SUCCESS: Query Code Compilation: {} -> {}",
in_file, out_file)
<< std::endl;
logger.debug("SUCCESS: Query Code Compilation: {} -> {}", in_file,
out_file);
}
protected:
Logger logger;
};

View File

@ -7,16 +7,15 @@
#include "template_engine/engine.hpp"
#include "traverser/cpp_traverser.hpp"
#include "utils/string/file.hpp"
// TODO:
// * logger
#include <iostream>
#include "logging/default.hpp"
using std::string;
class CodeGenerator
{
public:
CodeGenerator() : logger(logging::log->logger("CodeGenerator")) {}
void generate_cpp(const std::string &query, const uint64_t stripped_hash,
const std::string &path)
{
@ -33,6 +32,7 @@ public:
try {
tree = compiler.syntax_tree(query);
} catch (const std::runtime_error &e) {
logger.error("Syntax error: {}", query);
throw QueryEngineException(std::string(e.what()));
}
@ -44,6 +44,7 @@ public:
} catch (const SemanticError &e) {
throw e;
} catch (const std::exception &e) {
logger.error("AST traversal error: {}", std::string(e.what()));
throw QueryEngineException("Unknown code generation error");
}
@ -54,12 +55,14 @@ public:
{"query", query},
{"code", cpp_traverser.code}});
// TODO: use logger, ifndef
std::cout << generated << std::endl;
logger.trace("generated code: {}", generated);
utils::write_file(generated, path);
}
protected:
Logger logger;
private:
template_engine::TemplateEngine template_engine;
ast::Ast tree;

View File

@ -19,10 +19,11 @@ auto return_query_action =
fmt::format("{} couldn't be found (RETURN clause).", entity));
}
if (element.is_entity_only()) {
code += code_line(code::print_properties, entity);
code += code_line(code::write_entity, entity);
} else if (element.is_projection()) {
auto &property = element.property;
code += code_line(code::print_property, entity, property);
code += code_line("// TODO: implement projection");
// auto &property = element.property;
// code += code_line(code::print_property, entity, property);
}
}

View File

@ -5,5 +5,5 @@
auto transaction_commit_action = [](CypherStateData &,
const QueryActionData &) -> std::string {
return code_line(code::transaction_commit) +
code_line(code::return_empty_result);
code_line(code::return_true);
};

View File

@ -1,13 +1,14 @@
#pragma once
#include "communication/communication.hpp"
#include "database/db.hpp"
#include "query_engine/query_result.hpp"
#include "query_engine/query_stripped.hpp"
class ICodeCPU
{
public:
virtual QueryResult::sptr run(Db &db, code_args_t &args) = 0;
virtual bool run(Db &db, code_args_t &args,
communication::OutputStream &stream) = 0;
virtual ~ICodeCPU() {}
};

View File

@ -3,10 +3,9 @@
#include <string>
#include "database/db.hpp"
#include "query_engine/exceptions/exceptions.hpp"
#include "query_engine/util.hpp"
#include "query_program.hpp"
#include "utils/log/logger.hpp"
#include "query_engine/exceptions/exceptions.hpp"
// preparations before execution
// execution
@ -15,11 +14,12 @@
class ProgramExecutor
{
public:
auto execute(QueryProgram &program, Db& db)
auto execute(QueryProgram &program, Db &db,
communication::OutputStream &stream)
{
try {
// TODO: return result of query/code exection
return program.code->run(db, program.stripped.arguments);
return program.code->run(db, program.stripped.arguments, stream);
} catch (...) {
// TODO: return more information about the error
throw QueryEngineException("code execution error");

View File

@ -4,8 +4,6 @@
#include <string>
#include <unordered_map>
#define NOT_LOG_INFO
#include "config/config.hpp"
#include "query_engine/code_compiler.hpp"
#include "query_engine/code_generator.hpp"
@ -13,29 +11,28 @@
#include "query_engine/query_program.hpp"
#include "query_engine/query_stripper.hpp"
#include "utils/hashing/fnv.hpp"
#include "utils/log/logger.hpp"
#include "logging/default.hpp"
using std::string;
using std::cout;
using std::endl;
class ProgramLoader
{
public:
using sptr_code_lib = std::shared_ptr<CodeLib>;
ProgramLoader()
: stripper(make_query_stripper(TK_LONG, TK_FLOAT, TK_STR, TK_BOOL))
ProgramLoader() :
stripper(make_query_stripper(TK_LONG, TK_FLOAT, TK_STR, TK_BOOL)),
logger(logging::log->logger("ProgramLoader"))
{
}
auto load(const string &query)
{
auto stripped = stripper.strip(query);
LOG_INFO("stripped_query = " + stripped.query);
logger.debug("stripped_query = {}", stripped.query);
auto hash_string = std::to_string(stripped.hash);
LOG_INFO("query_hash = " + hash_string);
logger.debug("query_hash = {}", hash_string);
auto code_lib_iter = code_libs.find(stripped.hash);
@ -65,6 +62,9 @@ public:
return QueryProgram(code_lib->instance(), std::move(stripped));
}
protected:
Logger logger;
private:
// TODO somehow remove int.. from here
QueryStripper<int, int, int, int> stripper;

View File

@ -1,31 +1,45 @@
#pragma once
#include "database/db.hpp"
#include "logging/default.hpp"
#include "program_executor.hpp"
#include "program_loader.hpp"
#include "query_result.hpp"
#include "database/db.hpp"
//
// Current arhitecture:
// query -> code_loader -> query_stripper -> [code_generator]
// -> [code_compiler] -> code_executor
/*
* Current arhitecture:
* query -> code_loader -> query_stripper -> [code_generator]
* -> [code_compiler] -> code_executor
*/
// TODO
// * query engine will get a pointer to currently active database
// * TCP server session will have pointer to dbms and currently active
// * database
class QueryEngine
{
public:
auto execute(const std::string &query, Db& db)
QueryEngine() : logger(logging::log->logger("QueryEngine")) {}
auto execute(const std::string &query, Db &db,
communication::OutputStream &stream)
{
// TODO: error handling
auto program = program_loader.load(query);
auto result = program_executor.execute(program, db);
return result;
try {
auto program = program_loader.load(query);
auto result = program_executor.execute(program, db, stream);
if (UNLIKELY(!result)) {
// info because it might be something like deadlock in which
// case one thread is stopped and user has try again
logger.info(
"Unable to execute query (executor returned false)");
}
return result;
} catch (QueryEngineException &e) {
// in this case something fatal went wrong
logger.error("QueryEngineException: {}", std::string(e.what()));
return false;
}
}
protected:
Logger logger;
private:
ProgramExecutor program_executor;
ProgramLoader program_loader;

View File

@ -1,5 +1,8 @@
#pragma once
// !! DEPRICATED !!
// TODO: DELETE
#include <memory>
#include <string>
#include <unordered_map>

View File

@ -50,8 +50,13 @@ const std::string match_edge_by_id =
"auto {0} = db.graph.edges.find(t, args[{1}]->as<Int64>().value);\n"
" if (!{0}) return t.commit(), std::make_shared<QueryResult>();";
const std::string return_empty_result =
"return std::make_shared<QueryResult>();";
const std::string write_entity =
"stream.write_field(\"{0}\");\n"
" stream.write_record();\n"
" stream.write({0});\n"
" stream.write_success_empty();\n";
const std::string return_true = "return true;";
const std::string update_property = "{}.property(\"{}\", args[{}]);";

View File

@ -1,58 +0,0 @@
#pragma once
#include <string>
#include <iostream>
#include <ctime>
#include <iomanip>
namespace logger
{
class Logger
{
public:
Logger(Logger& other) = delete;
Logger(Logger&& other) = delete;
private:
Logger() = default;
// TODO logger name support
// TODO level support
// TODO handlers support:
// * log format support
// TODO merge with debug/log.hpp
public:
static Logger& instance()
{
static Logger logger;
return logger;
}
void info(const std::string& text)
{
stdout_log(text);
}
private:
void stdout_log(const std::string& text)
{
auto now = std::time(nullptr);
std::cout << std::put_time(std::gmtime(&now), "[%F %T]: ")
<< text << std::endl;
}
};
}
#ifdef NOT_LOG_INFO
# define LOG_INFO(_)
#else
# define LOG_INFO(_MESSAGE_) logger::Logger::instance().info(_MESSAGE_);
#endif

View File

@ -6,7 +6,7 @@
#include <thread>
#include <atomic>
#include "utils/log/logger.hpp"
#include "logging/default.hpp"
/** @class Timer
* @brief The timer contains counter and handler.
@ -116,7 +116,7 @@ public:
while (is_running.load()) {
std::this_thread::sleep_for(delta_time_type(delta_time));
timer_container.process();
LOG_INFO("timer_container.process()");
logging::info("timer_container.process()");
}
});
}

View File

@ -1,19 +0,0 @@
#include "bolt/v1/states.hpp"
#include "bolt/v1/states/handshake.hpp"
#include "bolt/v1/states/init.hpp"
#include "bolt/v1/states/error.hpp"
#include "bolt/v1/states/executor.hpp"
namespace bolt
{
States::States()
{
handshake = std::make_unique<Handshake>();
init = std::make_unique<Init>();
executor = std::make_unique<Executor>();
error = std::make_unique<Error>();
}
}

View File

@ -1,7 +1,6 @@
#include "bolt/v1/bolt.hpp"
#include "communication/bolt/v1/bolt.hpp"
#include "bolt/v1/session.hpp"
#include <iostream>
#include "communication/bolt/v1/session.hpp"
namespace bolt
{

View File

@ -1,4 +1,4 @@
#include "bolt/v1/session.hpp"
#include "communication/bolt/v1/session.hpp"
namespace bolt
{

View File

@ -0,0 +1,19 @@
#include "communication/bolt/v1/states.hpp"
#include "communication/bolt/v1/states/handshake.hpp"
#include "communication/bolt/v1/states/init.hpp"
#include "communication/bolt/v1/states/error.hpp"
#include "communication/bolt/v1/states/executor.hpp"
namespace bolt
{
States::States()
{
handshake = std::make_unique<Handshake>();
init = std::make_unique<Init>();
executor = std::make_unique<Executor>();
error = std::make_unique<Error>();
}
}

View File

@ -1,4 +1,4 @@
#include "bolt/v1/states/error.hpp"
#include "communication/bolt/v1/states/error.hpp"
namespace bolt
{

View File

@ -1,5 +1,5 @@
#include "bolt/v1/states/executor.hpp"
#include "bolt/v1/messaging/codes.hpp"
#include "communication/bolt/v1/states/executor.hpp"
#include "communication/bolt/v1/messaging/codes.hpp"
namespace bolt
{
@ -54,11 +54,11 @@ void Executor::run(Session& session, Query& query)
{
logger.trace("[Run] '{}'", query.statement);
// auto &db = session.active_db();
// logger.info("[ActiveDB] '{}'", db.name());
// query_engine.execute(query.statement, db);
session.output_stream._write_test();
auto &db = session.active_db();
logger.debug("[ActiveDB] '{}'", db.name());
// TODO: hangle syntax error use case
query_engine.execute(query.statement, db, session.output_stream);
}
void Executor::pull_all(Session& session)

View File

@ -1,6 +1,6 @@
#include "bolt/v1/states/handshake.hpp"
#include "communication/bolt/v1/states/handshake.hpp"
#include "bolt/v1/session.hpp"
#include "communication/bolt/v1/session.hpp"
namespace bolt
{

View File

@ -1,7 +1,7 @@
#include "bolt/v1/states/init.hpp"
#include "communication/bolt/v1/states/init.hpp"
#include "bolt/v1/session.hpp"
#include "bolt/v1/messaging/codes.hpp"
#include "communication/bolt/v1/session.hpp"
#include "communication/bolt/v1/messaging/codes.hpp"
#include "utils/likely.hpp"

View File

@ -1,10 +1,8 @@
#include "bolt/v1/transport/bolt_decoder.hpp"
#include <iostream>
#include "communication/bolt/v1/transport/bolt_decoder.hpp"
#include "communication/bolt/v1/packing/codes.hpp"
#include "logging/default.hpp"
#include "utils/bswap.hpp"
#include "bolt/v1/packing/codes.hpp"
namespace bolt
{

View File

@ -1,4 +1,4 @@
#include "bolt/v1/transport/buffer.hpp"
#include "communication/bolt/v1/transport/buffer.hpp"
namespace bolt
{

View File

@ -0,0 +1 @@
// TODO

View File

@ -0,0 +1 @@
// TODO

View File

@ -4,6 +4,7 @@
#include "token.hpp"
#include "ast/tree.hpp"
#include "tokenizer/cypher_lexer.hpp"
#include "logging/default.hpp"
void* cypher_parserAlloc(void* (*allocProc)(size_t));
void cypher_parser(void*, int, Token*, ast::Ast* ast);
@ -15,7 +16,7 @@ namespace cypher
class Parser
{
public:
Parser()
Parser() : logger(logging::log->logger("LexicalParser"))
{
parser = cypher_parserAlloc(malloc);
}
@ -34,7 +35,8 @@ public:
{
tokens.emplace_back(tokenizer.lookup());
auto& token = tokens.back();
std::cout << token << std::endl;
// TODO: resolve fmt error with {
// logger.debug(token.repr());
cypher_parser(parser, token.id, &token, &tree);
} while(tokens.back().id != 0);
@ -42,6 +44,9 @@ public:
return std::move(tree);
}
protected:
Logger logger;
private:
void* parser;
};

View File

@ -1,9 +1,10 @@
#pragma once
#include <iostream>
#include <ostream>
#include <cstdint>
#include <string>
#include <fmt/format.h>
struct Token
{
@ -25,6 +26,16 @@ struct Token
return id > 0;
}
/*
* String representation.
*/
std::string repr() const
{
// TODO: wrap fmt format
// return fmt::format("TOKEN id = {}, value = {}", id, value);
return "";
}
/*
* Ostream operator
*
@ -32,7 +43,6 @@ struct Token
*/
friend std::ostream& operator<<(std::ostream& stream, const Token& token)
{
return stream << "TOKEN id = " << token.id
<< ", value = '" << token.value << "'";
return stream << token.repr();
}
};

Binary file not shown.

View File

@ -1,13 +1,38 @@
from neo4j.v1 import GraphDatabase, basic_auth
from neo4j.v1 import GraphDatabase, basic_auth, types
# create session
driver = GraphDatabase.driver("bolt://localhost",
auth=basic_auth("neo4j", "neo4j"),
encrypted=0)
session = driver.session()
# session.run("CREATE (a:Person {age:25})")
result = session.run("MATCH (a:Person) RETURN a.name AS name")
for record in result:
print(record["name"])
session.close()
queries = [];
queries.append((True, "CREATE (n {name: \"Max\", age: 21}) RETURN n"))
queries.append((False, "CREATE (n {name: \"Paul\", age: 21}) RETURN n"))
queries.append((False, "CREATE (n:PERSON {name: \"Chris\", age: 20}) RETURN n"))
queries.append((False, "CREATE (n:PERSON:STUDENT {name: \"Marko\", age: 19}) RETURN n"))
queries.append((False, "CREATE (n:TEST {string: \"Properties test\", integer: 100, float: 232.2323, bool: True}) RETURN n"))
queries.append((False, "MATCH (n) WHERE ID(n)=0 RETURN n"))
queries.append((False, "MATCH (n) WHERE ID(n)=1 RETURN n"))
queries.append((False, "MATCH (n) WHERE ID(n)=2 RETURN n"))
queries.append((False, "MATCH (n) WHERE ID(n)=3 RETURN n"))
queries.append((False, "MATCH (n) WHERE ID(n)=4 RETURN n"))
queries.append((False, "MATCH (n1), (n2) WHERE ID(n1)=0 AND ID(n2)=1 CREATE (n1)-[r:IS]->(n2) RETURN r"))
queries.append((False, "MATCH (n1), (n2) WHERE ID(n1)=0 AND ID(n2)=1 CREATE (n1)-[r:IS {name: \"test\", age: 23}]->(n2) RETURN r"))
queries.append((False, "MATCH (n1), (n2) WHERE ID(n1)=0 AND ID(n2)=1 CREATE (n1)-[r:IS {name: \"test\", age: 23}]->(n2) RETURN r"))
queries.append((False, "MATCH ()-[r]-() WHERE ID(r)=0 RETURN r"))
queries.append((False, "MATCH ()-[r]-() WHERE ID(r)=1 RETURN r"))
queries.append((False, "MATCH ()-[r]-() WHERE ID(r)=2 RETURN r"))
queries.append((False, "MATCH (n) WHERE ID(n)=1 SET n.name = \"updated_name\" RETURN n"))
queries.append((False, "MATCH (n) WHERE ID(n)=1 RETURN n"))
queries.append((False, "MATCH ()-[r]-() WHERE ID(r)=1 SET r.name = \"TEST100\" RETURN r"))
queries.append((False, "MATCH ()-[r]-() WHERE ID(r)=1 RETURN r"))
for active, query in queries:
if active:
session.run(query)

View File

@ -1,5 +1,5 @@
#include "tls.hpp"
#include "tls_error.hpp"
#include "io/network/tls.hpp"
#include "io/network/tls_error.hpp"
namespace io
{

View File

@ -1,6 +1,6 @@
#pragma once
#include "tcpstream.hpp"
#include "io/uv/tcpstream.hpp"
namespace uv
{

View File

@ -3,7 +3,7 @@
#include <cstdlib>
#include <cstring>
#include "uvbuffer.hpp"
#include "io/uv/uvbuffer.hpp"
namespace uv
{

View File

@ -20,6 +20,16 @@ Logger init_debug_logger()
Logger debug_logger = init_debug_logger();
std::unique_ptr<Log> info_log = std::make_unique<SyncLog>();
Logger init_info_logger()
{
info_log->pipe(std::make_unique<Stdout>());
return info_log->logger("INFO");
}
Logger info_logger = init_info_logger();
void init_async()
{
log = std::make_unique<AsyncLog>();

View File

@ -1,8 +1,8 @@
#include <iostream>
#include <signal.h>
#include "bolt/v1/server/server.hpp"
#include "bolt/v1/server/worker.hpp"
#include "communication/bolt/v1/server/server.hpp"
#include "communication/bolt/v1/server/worker.hpp"
#include "io/network/socket.hpp"
@ -34,7 +34,7 @@ int main(void)
// that are configured below
std::set_terminate(&terminate_handler);
logging::init_sync();
logging::init_async();
logging::log->pipe(std::make_unique<Stdout>());
logger = logging::log->logger("Main");

View File

@ -14,9 +14,11 @@ class {{class_name}} : public ICodeCPU
{
public:
QueryResult::sptr run(Db& db, code_args_t& args) override
bool run(Db& db, code_args_t& args,
communication::OutputStream& stream) override
{
{{code}} }
{{code}}
}
~{{class_name}}() {}
};

View File

@ -63,19 +63,20 @@ endforeach()
# test hard coded queries
add_executable(integration_queries integration/queries.cpp)
target_link_libraries(integration_queries memgraph)
target_link_libraries(integration_queries Threads::Threads)
target_link_libraries(integration_queries ${fmt_static_lib})
add_test(NAME integration_queries COMMAND integration_queries)
set_property(TARGET integration_queries PROPERTY CXX_STANDARD 14)
# test query engine
add_executable(integration_query_engine integration/query_engine.cpp)
# target_link_libraries()
target_link_libraries(integration_query_engine Threads::Threads)
add_test(NAME integration_query_engine COMMAND integration_query_engine)
set_property(TARGET integration_query_engine PROPERTY CXX_STANDARD 14)
# test memgraph with bolt protocol
add_executable(integration_memgraph_bolt integration/memgraph_bolt.cpp)
# target_link_libraries()
target_link_libraries(integration_memgraph_bolt Threads::Threads)
add_test(NAME integration_memgraph_bolt COMMAND integration_memgraph_bolt)
set_property(TARGET integration_memgraph_bolt PROPERTY CXX_STANDARD 14)
@ -84,6 +85,7 @@ set_property(TARGET integration_memgraph_bolt PROPERTY CXX_STANDARD 14)
# cypher_ast
add_executable(manual_cypher_ast manual/cypher_ast.cpp)
target_link_libraries(manual_cypher_ast memgraph)
target_link_libraries(manual_cypher_ast Threads::Threads)
target_link_libraries(manual_cypher_ast ${fmt_static_lib})
target_link_libraries(manual_cypher_ast cypher_lib)
set_property(TARGET manual_cypher_ast PROPERTY CXX_STANDARD 14)
@ -91,6 +93,7 @@ set_property(TARGET manual_cypher_ast PROPERTY CXX_STANDARD 14)
# queries
add_executable(manual_queries manual/queries.cpp)
target_link_libraries(manual_queries memgraph)
target_link_libraries(manual_queries Threads::Threads)
target_link_libraries(manual_queries ${fmt_static_lib})
target_link_libraries(manual_queries cypher_lib)
set_property(TARGET manual_queries PROPERTY CXX_STANDARD 14)
@ -101,6 +104,7 @@ target_link_libraries(manual_query_engine memgraph)
target_link_libraries(manual_query_engine ${fmt_static_lib})
target_link_libraries(manual_query_engine dl)
target_link_libraries(manual_query_engine cypher_lib)
target_link_libraries(manual_query_engine Threads::Threads)
set_property(TARGET manual_query_engine PROPERTY CXX_STANDARD 14)
# query_hasher

View File

@ -1,7 +1,7 @@
#include <iostream>
#include <chrono>
#include "utils/log/logger.hpp"
#include "logging/default.cpp"
#include "utils/timer/timer.hpp"
using namespace std::chrono_literals;
@ -9,7 +9,7 @@ using namespace std::chrono_literals;
Timer::sptr create_test_timer(int64_t counter)
{
return std::make_shared<Timer>(
counter, [](){ LOG_INFO("Timer timeout"); }
counter, [](){ logging::info("Timer timeout"); }
);
}

View File

@ -7,6 +7,7 @@
#include "query_engine/query_engine.hpp"
#include "utils/time/timer.hpp"
#include "utils/terminate_handler.hpp"
#include "communication/communication.hpp"
using std::cout;
using std::endl;
@ -18,6 +19,9 @@ int main(void)
Db db;
QueryEngine engine;
// TODO: write dummy socket that is going to execute test
io::Socket socket;
communication::OutputStream stream(socket);
cout << "-- Memgraph query engine --" << endl;
@ -31,7 +35,7 @@ int main(void)
// execute command
try {
engine.execute(command, db);
engine.execute(command, db, stream);
} catch (const std::exception& e) {
cout << e.what() << endl;
} catch (const QueryEngineException& e) {

View File

@ -5,7 +5,7 @@
#include <array>
#include <vector>
#include "bolt/v1/transport/chunked_decoder.hpp"
#include "communication/bolt/v1/transport/chunked_decoder.hpp"
using byte = unsigned char;

View File

@ -3,7 +3,7 @@
#include <cassert>
#include <vector>
#include "bolt/v1/transport/chunked_encoder.hpp"
#include "communication/bolt/v1/transport/chunked_encoder.hpp"
using byte = unsigned char;