tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Load configs from flagfiles.

Browse Source 
Summary: Migrate configto gflags.

Reviewers: buda

Reviewed By: buda

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D433
This commit is contained in:
Dominik Gleich 2017-06-07 15:23:08 +02:00
parent 46581bfc94
commit 65507da9eb
22 changed files with 188 additions and 291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -1,6 +1,7 @@
# Change Log
## Next Build
* Abandon old config and migrate to gflags.
### Major Features and Improvements

2
CMakeLists.txt
View File

@ -334,7 +334,6 @@ target_link_libraries(antlr_opencypher_parser_lib antlr4_static)
# all memgraph src files
set(memgraph_src_files
${src_dir}/config/config.cpp
${src_dir}/dbms/dbms.cpp
# ${src_dir}/dbms/cleaner.cpp
${src_dir}/utils/numerics/saturate.cpp
@ -370,6 +369,7 @@ set(memgraph_src_files
${src_dir}/database/graph_db_accessor.cpp
${src_dir}/data_structures/concurrent/skiplist_gc.cpp
${src_dir}/query/stripper.cpp
${src_dir}/query/engine.cpp
${src_dir}/query/console.cpp
${src_dir}/query/frontend/ast/cypher_main_visitor.cpp
${src_dir}/query/typed_value.cpp

28
config/compiler.yaml
View File

@ -1,28 +0,0 @@
###########################
# MEMGRAPH DEFAULT CONFIG #
###########################
# NOTE: all paths are relative to the run folder
# (where the executable is runned)
# path to the codes which will be compiled
compile_path: "./compiled/"
# path to the template (cpp) for codes generation
template_cpp_path: "./template/plan_template_cpp"
# path to the folder with snapshots
snapshots_path: "snapshots"
# cleaning cycle interval
# if set to -1 the GC will not run
cleaning_cycle_sec: "30"
# snapshot cycle interval
snapshot_cycle_sec: "60"
# max number of snapshots which will be kept on the disk at some point
max_retained_snapshots: "3"
# by default query engine runs in interpret mode
interpret: false

28
config/hardcoded_queries_memgraph.yaml
View File

@ -1,28 +0,0 @@
###########################
# MEMGRAPH DEFAULT CONFIG #
###########################
# NOTE: all paths are relative to the run folder
# (where the executable is runned)
# path to the codes which will be compiled
compile_path: "./compiled/"
# path to the template (cpp) for codes generation
template_cpp_path: "./template/plan_template_cpp"
# path to the folder with snapshots
snapshots_path: "snapshots"
# cleaning cycle interval
# if set to -1 the GC will not run
cleaning_cycle_sec: "30"
# snapshot cycle interval
snapshot_cycle_sec: "60"
# max number of snapshots which will be kept on the disk at some point
max_retained_snapshots: "3"
# by default query engine runs in interpret mode
interpret: false

28
config/interpreter.yaml
View File

@ -1,28 +0,0 @@
###########################
# MEMGRAPH DEFAULT CONFIG #
###########################
# NOTE: all paths are relative to the run folder
# (where the executable is runned)
# path to the codes which will be compiled
compile_path: "./compiled/"
# path to the template (cpp) for codes generation
template_cpp_path: "./template/plan_template_cpp"
# path to the folder with snapshots
snapshots_path: "snapshots"
# cleaning cycle interval
# if set to -1 the GC will not run
cleaning_cycle_sec: "30"
# snapshot cycle interval
snapshot_cycle_sec: "60"
# max number of snapshots which will be kept on the disk at some point
max_retained_snapshots: "3"
# by default query engine runs in interpret mode
interpret: true

22
config/memgraph.yaml → config/testing
View File

@ -1,36 +1,34 @@
###########################
# MEMGRAPH DEFAULT CONFIG #
###########################
# MEMGRAPH DEFAULT TESTING CONFIG
# NOTE: all paths are relative to the run folder
# (where the executable is runned)
# path to the codes which will be compiled
compile_path: "./compiled/"
--compile_path="./compiled/"
# path to the template (cpp) for codes generation
template_cpp_path: "./template/plan_template_cpp"
--template_cpp_path="./template/plan_template_cpp"
# path to the folder with snapshots
snapshots_path: "snapshots"
--snapshots_path="snapshots"
# cleaning cycle interval
# if set to -1 the GC will not run
cleaning_cycle_sec: "30"
--cleaning_cycle_sec=30
# snapshot cycle interval
# if set to -1 the snapshooter will not run
snapshot_cycle_sec: "-1"
--snapshot_cycle_sec=-1
# create snapshot disabled on db destruction
snapshot_db_destruction: false
--snapshot_db_destruction=false
# max number of snapshots which will be kept on the disk at some point
# if set to -1 the max number of snapshots is unlimited
max_retained_snapshots: "-1"
--max_retained_snapshots=-1
# by default query engine runs in interpret mode
interpret: true
--interpret=true
# database recovering is disabled by default
recovery: false
--recovery=false

53
src/config/config.hpp
View File

@ -1,53 +0,0 @@
#pragma once
#include "utils/config/config.hpp"
#include <set>
#include <string>
namespace config {
// this class is used as a Definition class of config::Config class from utils
// number of elements should be small,
// it depends on implementation of config::Config class
// in other words number of fields in Definition class should be related
// to the number of config keys
class MemgraphConfig {
public:
static const char *env_config_key;
static const char *default_file_path;
static std::set<std::string> arguments;
};
// -- all possible Memgraph's keys --
constexpr const char *COMPILE_PATH = "compile_path";
constexpr const char *TEMPLATE_CPP_PATH = "template_cpp_path";
constexpr const char *SNAPSHOTS_PATH = "snapshots_path";
constexpr const char *CLEANING_CYCLE_SEC = "cleaning_cycle_sec";
constexpr const char *SNAPSHOT_CYCLE_SEC = "snapshot_cycle_sec";
constexpr const char *SNAPSHOT_DB_DESTRUCTION = "snapshot_db_destruction";
constexpr const char *MAX_RETAINED_SNAPSHOTS = "max_retained_snapshots";
constexpr const char *INTERPRET = "interpret";
constexpr const char *RECOVERY = "recovery";
// -- all possible Memgraph's keys --
inline long long to_int(const std::string &s) { return stoll(s); }
// TODO: move this to register args because it doesn't make sense to convert
// str to bool for every lookup
inline bool to_bool(std::string &s) {
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
std::istringstream is(s);
bool b;
is >> std::boolalpha >> b;
return b;
}
}
// code uses this define for key access
// _KEY_ is value from all possible keys that are listed above
#define CONFIG_REGISTER_ARGS(ARGC, ARGV) \
config::Config<config::MemgraphConfig>::instance().register_args(ARGC, ARGV);
#define CONFIG(_KEY_) config::Config<config::MemgraphConfig>::instance()[_KEY_]
#define CONFIG_INTEGER(_KEY_) config::to_int(CONFIG(_KEY_))
#define CONFIG_BOOL(_KEY_) config::to_bool(CONFIG(_KEY_))

62
src/database/graph_db.cpp
View File

@ -1,6 +1,7 @@
#include <functional>
#include "config/config.hpp"
#include "gflags/gflags.h"
#include "database/creation_exception.hpp"
#include "database/graph_db.hpp"
#include "database/graph_db_accessor.hpp"
@ -9,22 +10,27 @@
#include "storage/edge.hpp"
#include "storage/garbage_collector.hpp"
const int DEFAULT_CLEANING_CYCLE_SEC = 30; // 30 seconds
const std::string DEFAULT_SNAPSHOT_FOLDER = "snapshots";
const int DEFAULT_MAX_RETAINED_SNAPSHOTS = -1; // unlimited number of snapshots
const int DEFAULT_SNAPSHOT_CYCLE_SEC = -1; // off
DEFINE_int32(GC_CYCLE_SEC, 30,
"Amount of time between starts of two cleaning cycles in seconds. "
"-1 to turn off.");
DEFINE_int32(MAX_RETAINED_SNAPSHOTS, -1,
"Number of retained snapshots, -1 means without limit.");
DEFINE_int32(SNAPSHOT_CYCLE_SEC, -1,
"Amount of time between starts of two snapshooters in seconds. -1 "
"to turn off.");
DEFINE_bool(SNAPSHOT_ON_DB_DESTRUCTION, false,
"Snapshot on database destruction.");
DECLARE_string(SNAPSHOT_DIRECTORY);
GraphDb::GraphDb(const std::string &name, const fs::path &snapshot_db_dir)
: name_(name),
gc_vertices_(vertices_, vertex_record_deleter_,
vertex_version_list_deleter_),
gc_edges_(edges_, edge_record_deleter_, edge_version_list_deleter_) {
const std::string time_str = CONFIG(config::CLEANING_CYCLE_SEC);
int pause = DEFAULT_CLEANING_CYCLE_SEC;
if (!time_str.empty()) pause = CONFIG_INTEGER(config::CLEANING_CYCLE_SEC);
// Pause of -1 means we shouldn't run the GC.
if (pause != -1) {
gc_scheduler_.Run(std::chrono::seconds(pause), [this]() {
if (FLAGS_GC_CYCLE_SEC != -1) {
gc_scheduler_.Run(std::chrono::seconds(FLAGS_GC_CYCLE_SEC), [this]() {
// main garbage collection logic
// see wiki documentation for logic explanation
const auto next_id = this->tx_engine.count() + 1;
@ -51,36 +57,17 @@ GraphDb::GraphDb(const std::string &name, const fs::path &snapshot_db_dir)
RecoverDatabase(snapshot_db_dir);
StartSnapshooting();
}
void GraphDb::StartSnapshooting() {
const std::string max_retained_snapshots_str =
CONFIG(config::MAX_RETAINED_SNAPSHOTS);
const std::string snapshot_cycle_sec_str =
CONFIG(config::SNAPSHOT_CYCLE_SEC);
const std::string snapshot_folder_str = CONFIG(config::SNAPSHOTS_PATH);
max_retained_snapshots_ = DEFAULT_MAX_RETAINED_SNAPSHOTS;
if (!max_retained_snapshots_str.empty())
max_retained_snapshots_ = CONFIG_INTEGER(config::MAX_RETAINED_SNAPSHOTS);
snapshot_cycle_sec_ = DEFAULT_SNAPSHOT_CYCLE_SEC;
if (!snapshot_cycle_sec_str.empty())
snapshot_cycle_sec_ = CONFIG_INTEGER(config::SNAPSHOT_CYCLE_SEC);
snapshot_folder_ = DEFAULT_SNAPSHOT_FOLDER;
if (!snapshot_folder_str.empty()) snapshot_folder_ = snapshot_folder_str;
snapshot_db_destruction_ = CONFIG_BOOL(config::SNAPSHOT_DB_DESTRUCTION);
if (snapshot_cycle_sec_ != -1) {
if (FLAGS_SNAPSHOT_CYCLE_SEC != -1) {
auto create_snapshot = [this]() -> void {
GraphDbAccessor db_accessor(*this);
snapshooter_.MakeSnapshot(db_accessor, fs::path(snapshot_folder_) / name_,
max_retained_snapshots_);
snapshooter_.MakeSnapshot(db_accessor,
fs::path(FLAGS_SNAPSHOT_DIRECTORY) / name_,
FLAGS_MAX_RETAINED_SNAPSHOTS);
};
snapshot_creator_.Run(std::chrono::seconds(snapshot_cycle_sec_),
snapshot_creator_.Run(std::chrono::seconds(FLAGS_SNAPSHOT_CYCLE_SEC),
create_snapshot);
}
}
@ -110,10 +97,11 @@ GraphDb::~GraphDb() {
snapshot_creator_.Stop();
// Create last database snapshot
if (snapshot_db_destruction_) {
if (FLAGS_SNAPSHOT_ON_DB_DESTRUCTION == true) {
GraphDbAccessor db_accessor(*this);
snapshooter_.MakeSnapshot(db_accessor, fs::path(snapshot_folder_) / name_,
max_retained_snapshots_);
snapshooter_.MakeSnapshot(db_accessor,
fs::path(FLAGS_SNAPSHOT_DIRECTORY) / name_,
FLAGS_MAX_RETAINED_SNAPSHOTS);
}
// Delete vertices and edges which weren't collected before, also deletes

4
src/database/graph_db.hpp
View File

@ -104,10 +104,6 @@ class GraphDb {
// snapshooter
Snapshooter snapshooter_;
std::string snapshot_folder_;
int max_retained_snapshots_;
int snapshot_cycle_sec_;
bool snapshot_db_destruction_;
// Schedulers
Scheduler<std::mutex> gc_scheduler_;

6
src/dbms/dbms.cpp
View File

@ -1,5 +1,11 @@
#include "gflags/gflags.h"
#include "dbms/dbms.hpp"
DEFINE_string(SNAPSHOT_DIRECTORY, "snapshots",
"Relative path to directory in which to save snapshots.");
DEFINE_bool(RECOVER_ON_STARTUP, false, "Recover database on startup.");
std::unique_ptr<GraphDbAccessor> Dbms::active() {
return std::make_unique<GraphDbAccessor>(
*active_db.load(std::memory_order_acquire));

23
src/dbms/dbms.hpp
View File

@ -4,27 +4,28 @@
#include <memory>
#include <vector>
#include "config/config.hpp"
#include "gflags/gflags.h"
#include "data_structures/concurrent/concurrent_map.hpp"
#include "database/graph_db.hpp"
#include "database/graph_db_accessor.hpp"
#include "durability/recovery.hpp"
//#include "dbms/cleaner.hpp"
DECLARE_string(SNAPSHOT_DIRECTORY);
DECLARE_bool(RECOVER_ON_STARTUP);
namespace fs = std::experimental::filesystem;
const std::string DEFAULT_SNAPSHOT_FOLDER = "snapshots";
class Dbms {
public:
Dbms() {
if (CONFIG_BOOL(config::RECOVERY)) {
auto accessor = dbs.access();
std::string snapshot_folder = CONFIG(config::SNAPSHOTS_PATH);
if (snapshot_folder.empty()) snapshot_folder = DEFAULT_SNAPSHOT_FOLDER;
for (auto &snapshot_db : fs::directory_iterator(snapshot_folder)) {
// create db and set it active
active(snapshot_db.path().filename(), snapshot_db);
if (FLAGS_RECOVER_ON_STARTUP) {
if (fs::exists(fs::path(FLAGS_SNAPSHOT_DIRECTORY))) {
auto accessor = dbs.access();
for (auto &snapshot_db :
fs::directory_iterator(FLAGS_SNAPSHOT_DIRECTORY)) {
// create db and set it active
active(snapshot_db.path().filename(), snapshot_db);
}
}
}

5
src/durability/snapshooter.cpp
View File

@ -1,11 +1,12 @@
#include "durability/snapshooter.hpp"
#include <algorithm>
#include "communication/bolt/v1/encoder/base_encoder.hpp"
#include "config/config.hpp"
#include "database/graph_db_accessor.hpp"
#include "durability/file_writer_buffer.hpp"
#include "utils/datetime/timestamp.hpp"
#include "durability/snapshooter.hpp"
bool Snapshooter::MakeSnapshot(GraphDbAccessor &db_accessor_,
const fs::path &snapshot_folder,
const int max_retained_snapshots) {

47
src/memgraph_bolt.cpp
View File

@ -17,7 +17,6 @@
#include "logging/default.hpp"
#include "logging/streams/stdout.hpp"
#include "utils/config/config.hpp"
#include "utils/signals/handler.hpp"
#include "utils/stacktrace/log.hpp"
#include "utils/terminate_handler.hpp"
@ -46,9 +45,50 @@ void throw_and_stacktace(std::string message) {
logger.info(stacktrace.dump());
}
// Load flags in this order, the last one has the highest priority:
// 1) /etc/memgraph/config
// 2) ~/.memgraph/config
// 3) env - MEMGRAPH_CONFIG
// 4) command line flags
void load_config(int &argc, char **&argv) {
std::vector<fs::path> configs = {fs::path("/etc/memgraph/config")};
if (getenv("HOME") != nullptr)
configs.emplace_back(fs::path(getenv("HOME")) /
fs::path(".memgraph/config"));
if (getenv("MEMGRAPH_CONFIG") != nullptr)
configs.emplace_back(fs::path(getenv("MEMGRAPH_CONFIG")));
std::vector<std::string> flagfile_arguments;
for (const auto &config : configs)
if (fs::exists(config)) {
flagfile_arguments.emplace_back(
std::string("--flagfile=" + config.generic_string()));
}
int custom_argc = static_cast<int>(flagfile_arguments.size()) + 1;
char **custom_argv = new char *[custom_argc];
custom_argv[0] = strdup(std::string("memgraph").c_str());
for (int i = 0; i < (int)flagfile_arguments.size(); ++i) {
custom_argv[i + 1] = strdup(flagfile_arguments[i].c_str());
}
// setup flags from config flags
gflags::ParseCommandLineFlags(&custom_argc, &custom_argv, false);
// unconsumed arguments have to be freed to avoid memory leak since they are
// strdup-ed.
for (int i = 0; i < custom_argc; ++i) free(custom_argv[i]);
delete[] custom_argv;
// setup flags from command line
gflags::ParseCommandLineFlags(&argc, &argv, true);
}
int main(int argc, char **argv) {
fs::current_path(fs::path(argv[0]).parent_path());
gflags::ParseCommandLineFlags(&argc, &argv, true);
load_config(argc, argv);
// Logging init.
#ifdef SYNC_LOGGER
logging::init_sync();
@ -78,9 +118,6 @@ int main(int argc, char **argv) {
std::exit(EXIT_FAILURE);
});
// Register args.
CONFIG_REGISTER_ARGS(argc, argv);
// Initialize endpoint.
endpoint_t endpoint;
try {

6
src/query/engine.cpp Normal file
View File

@ -0,0 +1,6 @@
#include "query/engine.hpp"
DEFINE_bool(INTERPRET, true,
"Use interpretor instead of compiler for query execution.");
DEFINE_string(COMPILE_DIRECTORY, "./compiled/",
"Directory in which to write compiled libraries.");

12
src/query/engine.hpp
View File

@ -3,7 +3,6 @@
#include <experimental/filesystem>
namespace fs = std::experimental::filesystem;
#include "config/config.hpp"
#include "data_structures/concurrent/concurrent_map.hpp"
#include "database/graph_db.hpp"
#include "logging/loggable.hpp"
@ -15,6 +14,9 @@ namespace fs = std::experimental::filesystem;
#include "query/preprocessor.hpp"
#include "utils/dynamic_lib.hpp"
DECLARE_bool(INTERPRET);
DECLARE_string(COMPILE_DIRECTORY);
/**
* Responsible for query execution.
*
@ -64,7 +66,7 @@ class QueryEngine : public Loggable {
*/
auto Run(const std::string &query, GraphDbAccessor &db_accessor,
Stream &stream) {
if (CONFIG_BOOL(config::INTERPRET)) {
if (FLAGS_INTERPRET) {
query::Interpret(query, db_accessor, stream);
return true;
}
@ -155,13 +157,13 @@ class QueryEngine : public Loggable {
return query_plan_it->second->instance();
// find hardcoded query plan if exists
auto hardcoded_path = fs::path(CONFIG(config::COMPILE_PATH) + "hardcode/" +
auto hardcoded_path = fs::path(FLAGS_COMPILE_DIRECTORY + "hardcode/" +
std::to_string(stripped.hash) + ".cpp");
if (fs::exists(hardcoded_path))
return LoadCpp(hardcoded_path, stripped.hash);
// generate query plan
auto generated_path = fs::path(CONFIG(config::COMPILE_PATH) +
auto generated_path = fs::path(FLAGS_COMPILE_DIRECTORY +
std::to_string(stripped.hash) + ".cpp");
query::frontend::opencypher::Parser parser(stripped.query);
@ -194,7 +196,7 @@ class QueryEngine : public Loggable {
// and that is a problem because at this point we want brand new
// dynamic lib. That is the tmp solution. The right solution would be
// to deal with this problem in DynamicLib
auto path_so = CONFIG(config::COMPILE_PATH) + std::to_string(hash) + "_" +
auto path_so = FLAGS_COMPILE_DIRECTORY + std::to_string(hash) + "_" +
(std::string)Timestamp::now() + ".so";
plan_compiler.Compile(path_cpp, path_so);

8
tests/integration/query_engine.cpp
View File

@ -2,10 +2,12 @@
#include "gflags/gflags.h"
#include "config/config.hpp"
#include "dbms/dbms.hpp"
#include "query_engine_common.hpp"
DECLARE_bool(INTERPRET);
DECLARE_string(COMPILE_DIRECTORY);
using namespace std::chrono_literals;
using namespace tests::integration;
@ -30,10 +32,10 @@ int main(int argc, char *argv[]) {
auto log = init_logging("IntegrationQueryEngine");
// Manually set config compile_path to avoid loading whole config file with
// the test.
CONFIG(config::COMPILE_PATH) = "../compiled/";
FLAGS_COMPILE_DIRECTORY = "../compiled/";
// Set the interpret to false to avoid calling the interpreter which doesn't
// support all the queries yet.
CONFIG(config::INTERPRET) = "false";
FLAGS_INTERPRET = false;
Dbms dbms;
StreamT stream(std::cout);
QueryEngineT query_engine;

6
tests/manual/console_test.cpp
View File

@ -1,5 +1,7 @@
#include <iostream>
#include "gflags/gflags.h"
#include "dbms/dbms.hpp"
#include "query/console.hpp"
#include "query/interpreter.hpp"
@ -24,12 +26,12 @@ void random_generate(Dbms &dbms, uint node_count, int edge_factor = 5) {
}
int main(int argc, char *argv[]) {
REGISTER_ARGS(argc, argv);
gflags::ParseCommandLineFlags(&argc, &argv, true);
// parse the first cmd line argument as the count of nodes to randomly create
uint node_count = 0;
if (argc > 1) {
node_count = (uint) std::stoul(argv[1]);
node_count = (uint)std::stoul(argv[1]);
permanent_assert(node_count < 10000000,
"More then 10M nodes requested, that's too much");
}

64
tests/unit/bolt_session.cpp
View File

@ -1,18 +1,19 @@
#include "bolt_common.hpp"
#include "gflags/gflags.h"
#include "bolt_common.hpp"
#include "communication/bolt/v1/encoder/result_stream.hpp"
#include "communication/bolt/v1/session.hpp"
#include "config/config.hpp"
#include "query/engine.hpp"
DECLARE_bool(INTERPRET);
// Shortcuts for writing variable initializations in tests
#define INIT_VARS Dbms dbms;\
TestSocket socket(10);\
QueryEngine<ResultStreamT> query_engine;\
SessionT session(std::move(socket), dbms, query_engine);\
std::vector<uint8_t> &output = session.socket_.output;
#define INIT_VARS \
Dbms dbms; \
TestSocket socket(10); \
QueryEngine<ResultStreamT> query_engine; \
SessionT session(std::move(socket), dbms, query_engine); \
std::vector<uint8_t> &output = session.socket_.output;
using ResultStreamT =
communication::bolt::ResultStream<communication::bolt::Encoder<
@ -20,19 +21,18 @@ using ResultStreamT =
using SessionT = communication::bolt::Session<TestSocket>;
using StateT = communication::bolt::State;
// Sample testdata that has correct inputs and outputs.
const uint8_t handshake_req[] = {
0x60, 0x60, 0xb0, 0x17, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
const uint8_t handshake_req[] = {0x60, 0x60, 0xb0, 0x17, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
const uint8_t handshake_resp[] = {0x00, 0x00, 0x00, 0x01};
const uint8_t init_req[] = {
0xb2, 0x01, 0xd0, 0x15, 0x6c, 0x69, 0x62, 0x6e, 0x65, 0x6f, 0x34, 0x6a,
0x2d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x31, 0x2e, 0x32, 0x2e,
0x31, 0xa3, 0x86, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x65, 0x85, 0x62, 0x61,
0x73, 0x69, 0x63, 0x89, 0x70, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61,
0x6c, 0x80, 0x8b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
0x6c, 0x73, 0x80};
0xb2, 0x01, 0xd0, 0x15, 0x6c, 0x69, 0x62, 0x6e, 0x65, 0x6f, 0x34,
0x6a, 0x2d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x31, 0x2e,
0x32, 0x2e, 0x31, 0xa3, 0x86, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x65,
0x85, 0x62, 0x61, 0x73, 0x69, 0x63, 0x89, 0x70, 0x72, 0x69, 0x6e,
0x63, 0x69, 0x70, 0x61, 0x6c, 0x80, 0x8b, 0x63, 0x72, 0x65, 0x64,
0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x80};
const uint8_t init_resp[] = {0x00, 0x03, 0xb1, 0x70, 0xa0, 0x00, 0x00};
const uint8_t run_req_header[] = {0xb2, 0x10, 0xd1};
const uint8_t pullall_req[] = {0xb0, 0x3f};
@ -42,7 +42,6 @@ const uint8_t ackfailure_req[] = {0xb0, 0x0e};
const uint8_t success_resp[] = {0x00, 0x03, 0xb1, 0x70, 0xa0, 0x00, 0x00};
const uint8_t ignored_resp[] = {0x00, 0x02, 0xb0, 0x7e, 0x00, 0x00};
// Write bolt chunk header (length)
void WriteChunkHeader(SessionT &session, uint16_t len) {
len = bswap(len);
@ -52,16 +51,14 @@ void WriteChunkHeader(SessionT &session, uint16_t len) {
}
// Write bolt chunk tail (two zeros)
void WriteChunkTail(SessionT &session) {
WriteChunkHeader(session, 0);
}
void WriteChunkTail(SessionT &session) { WriteChunkHeader(session, 0); }
// Check that the server responded with a failure message
void CheckFailureMessage(std::vector<uint8_t> &output) {
ASSERT_GE(output.size(), 6);
// skip the first two bytes because they are the chunk header
ASSERT_EQ(output[2], 0xB1); // tiny struct 1
ASSERT_EQ(output[3], 0x7F); // signature failure
ASSERT_EQ(output[2], 0xB1); // tiny struct 1
ASSERT_EQ(output[3], 0x7F); // signature failure
}
// Execute and check a correct handshake
@ -78,7 +75,8 @@ void ExecuteHandshake(SessionT &session, std::vector<uint8_t> &output) {
}
// Write bolt chunk and execute command
void ExecuteCommand(SessionT &session, const uint8_t *data, size_t len, bool chunk = true) {
void ExecuteCommand(SessionT &session, const uint8_t *data, size_t len,
bool chunk = true) {
if (chunk) WriteChunkHeader(session, len);
auto buff = session.Allocate();
memcpy(buff.data, data, len);
@ -117,21 +115,19 @@ void WriteRunRequest(SessionT &session, const char *str) {
// write empty map for parameters
buff = session.Allocate();
buff.data[0] = 0xA0; // TinyMap0
buff.data[0] = 0xA0; // TinyMap0
session.Written(1);
// write chunk tail
WriteChunkTail(session);
}
TEST(BoltSession, HandshakeWrongPreamble) {
INIT_VARS;
auto buff = session.Allocate();
// copy 0x00000001 four times
for (int i = 0; i < 4; ++i)
memcpy(buff.data + i * 4, handshake_req + 4, 4);
for (int i = 0; i < 4; ++i) memcpy(buff.data + i * 4, handshake_req + 4, 4);
session.Written(20);
session.Execute();
@ -162,7 +158,6 @@ TEST(BoltSession, HandshakeInTwoPackets) {
CheckOutput(output, handshake_resp, 4);
}
TEST(BoltSession, HandshakeWriteFail) {
INIT_VARS;
session.socket_.SetWriteSuccess(false);
@ -178,7 +173,6 @@ TEST(BoltSession, HandshakeOK) {
ExecuteHandshake(session, output);
}
TEST(BoltSession, InitWrongSignature) {
INIT_VARS;
ExecuteHandshake(session, output);
@ -235,7 +229,6 @@ TEST(BoltSession, InitOK) {
ExecuteInit(session, output);
}
TEST(BoltSession, ExecuteRunWrongMarker) {
INIT_VARS;
@ -457,7 +450,7 @@ TEST(BoltSession, ErrorOK) {
const uint8_t *dataset[] = {ackfailure_req, reset_req};
for (int i = 0; i < 2; ++i) {
// first test with socket write success, then with socket write fail
// first test with socket write success, then with socket write fail
for (int j = 0; j < 2; ++j) {
INIT_VARS;
@ -523,7 +516,7 @@ TEST(BoltSession, MultipleChunksInOneExecute) {
// Count chunks in output
int len, num = 0;
while(output.size() > 0) {
while (output.size() > 0) {
len = (output[0] << 8) + output[1];
output.erase(output.begin(), output.begin() + len + 4);
++num;
@ -579,13 +572,12 @@ TEST(BoltSession, InvalidChunk) {
CheckFailureMessage(output);
}
int main(int argc, char **argv) {
logging::init_sync();
logging::log->pipe(std::make_unique<Stdout>());
// Set the interpret to true to avoid calling the compiler which only
// supports a limited set of queries.
CONFIG(config::INTERPRET) = "true";
FLAGS_INTERPRET = true;
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

34
tests/unit/dbms_recovery.cpp
View File

@ -1,7 +1,14 @@
#include <experimental/filesystem>
#include "dbms/dbms.hpp"
#include "gflags/gflags.h"
#include "gtest/gtest.h"
#include "dbms/dbms.hpp"
DECLARE_bool(RECOVERY_ON_STARTUP);
DECLARE_string(SNAPSHOT_DIRECTORY);
DECLARE_int32(SNAPSHOT_CYCLE_SEC);
namespace fs = std::experimental::filesystem;
const fs::path SNAPSHOTS_DBMS_RECOVERY_ALL_DB = std::tmpnam(nullptr);
@ -25,25 +32,17 @@ void CleanDbDir() {
class DbmsRecoveryTest : public ::testing::Test {
protected:
virtual void TearDown() {
CleanDbDir();
CONFIG(config::SNAPSHOTS_PATH) = snapshots_path_setup_;
CONFIG(config::SNAPSHOT_CYCLE_SEC) = snapshot_cycle_sec_setup_;
}
virtual void TearDown() { CleanDbDir(); }
virtual void SetUp() {
CleanDbDir();
snapshots_path_setup_ = CONFIG(config::SNAPSHOTS_PATH);
snapshot_cycle_sec_setup_ = CONFIG(config::SNAPSHOT_CYCLE_SEC);
CONFIG(config::SNAPSHOTS_PATH) = SNAPSHOTS_DBMS_RECOVERY_ALL_DB;
CONFIG(config::SNAPSHOT_CYCLE_SEC) = "-1";
FLAGS_SNAPSHOT_DIRECTORY = SNAPSHOTS_DBMS_RECOVERY_ALL_DB;
FLAGS_SNAPSHOT_CYCLE_SEC = -1;
}
std::string snapshots_path_setup_;
std::string snapshot_cycle_sec_setup_;
};
void CreateSnapshot() {
CONFIG(config::RECOVERY) = "false";
FLAGS_RECOVER_ON_STARTUP = false;
Dbms dbms;
auto dba = dbms.active();
@ -56,12 +55,13 @@ void CreateSnapshot() {
dba->advance_command();
Snapshooter snapshooter;
snapshooter.MakeSnapshot(*dba.get(), SNAPSHOTS_DBMS_RECOVERY_DEFAULT_DB_DIR,
1);
EXPECT_EQ(snapshooter.MakeSnapshot(*dba.get(),
SNAPSHOTS_DBMS_RECOVERY_DEFAULT_DB_DIR, 1),
true);
}
void RecoverDbms() {
CONFIG(config::RECOVERY) = "true";
FLAGS_RECOVER_ON_STARTUP = true;
Dbms dbms;
auto dba = dbms.active();
@ -84,7 +84,7 @@ void RecoverDbms() {
edges.push_back(edge);
edge_count++;
}
EXPECT_EQ(edge_count, 2);
ASSERT_EQ(edge_count, 2);
EXPECT_EQ(edges[0].to() == edges[1].to(), true);
EXPECT_EQ(edges[0].from() == edges[1].from(), false);
}

1
tests/unit/mvcc_gc.cpp
View File

@ -5,7 +5,6 @@
#include <memory>
#include <thread>
#include "config/config.hpp"
#include "data_structures/concurrent/skiplist.hpp"
#include "logging/logger.hpp"
#include "logging/streams/stdout.hpp"

18
tests/unit/recovery.cpp
View File

@ -1,12 +1,17 @@
#include "durability/recovery.hpp"
#include <cstdio>
#include <experimental/filesystem>
#include "gflags/gflags.h"
#include "gtest/gtest.h"
#include "communication/bolt/v1/decoder/decoder.hpp"
#include "dbms/dbms.hpp"
#include "durability/file_reader_buffer.hpp"
#include "gtest/gtest.h"
#include "durability/recovery.hpp"
#include "utils/assert.hpp"
DECLARE_int32(SNAPSHOT_CYCLE_SEC);
namespace fs = std::experimental::filesystem;
const fs::path SNAPSHOTS_RECOVERY_DEFAULT_DB_DIR = std::tmpnam(nullptr);
@ -30,17 +35,12 @@ void CleanDbDir() {
class RecoveryTest : public ::testing::Test {
protected:
void TearDown() override {
CleanDbDir();
CONFIG(config::SNAPSHOT_CYCLE_SEC) = snapshot_cycle_sec_setup_;
}
void TearDown() override { CleanDbDir(); }
void SetUp() override {
CleanDbDir();
snapshot_cycle_sec_setup_ = CONFIG(config::SNAPSHOT_CYCLE_SEC);
CONFIG(config::SNAPSHOT_CYCLE_SEC) = "-1";
FLAGS_SNAPSHOT_CYCLE_SEC = -1;
}
std::string snapshot_cycle_sec_setup_;
const int max_retained_snapshots_ = 10;
};

21
tests/unit/snapshot.cpp
View File

@ -1,7 +1,14 @@
#include <experimental/filesystem>
#include "gflags/gflags.h"
#include "gtest/gtest.h"
#include "dbms/dbms.hpp"
#include "durability/snapshooter.hpp"
#include "gtest/gtest.h"
DECLARE_bool(SNAPSHOT_ON_DB_DESTRUCTION);
DECLARE_int32(SNAPSHOT_CYCLE_SEC);
DECLARE_string(SNAPSHOT_DIRECTORY);
namespace fs = std::experimental::filesystem;
@ -28,15 +35,11 @@ void CleanDbDir() {
class SnapshotTest : public ::testing::Test {
protected:
virtual void TearDown() {
CleanDbDir();
CONFIG(config::SNAPSHOT_CYCLE_SEC) = snapshot_cycle_sec_setup_;
}
virtual void TearDown() { CleanDbDir(); }
virtual void SetUp() {
CleanDbDir();
snapshot_cycle_sec_setup_ = CONFIG(config::SNAPSHOT_CYCLE_SEC);
CONFIG(config::SNAPSHOT_CYCLE_SEC) = "-1";
FLAGS_SNAPSHOT_CYCLE_SEC = -1;
}
std::string snapshot_cycle_sec_setup_;
};
@ -97,8 +100,8 @@ TEST_F(SnapshotTest, CreateSnapshotWithUnlimitedMaxRetainedSnapshots) {
TEST_F(SnapshotTest, TestSnapshotFileOnDbDestruct) {
{
CONFIG(config::SNAPSHOTS_PATH) = SNAPSHOTS_FOLDER_ALL_DB;
CONFIG(config::SNAPSHOT_DB_DESTRUCTION) = "true";
FLAGS_SNAPSHOT_DIRECTORY = SNAPSHOTS_FOLDER_ALL_DB;
FLAGS_SNAPSHOT_ON_DB_DESTRUCTION = true;
Dbms dbms;
auto dba = dbms.active();
}