diff --git a/.ycm_extra_conf.py b/.ycm_extra_conf.py index f29ea8585..99b8f46ff 100644 --- a/.ycm_extra_conf.py +++ b/.ycm_extra_conf.py @@ -32,6 +32,9 @@ BASE_FLAGS = [ '-I./libs/gflags/include', '-I./experimental/distributed/src', '-I./libs/postgresql/include', + '-I./libs/bzip2', + '-I./libs/zlib', + '-I./libs/rocksdb/include', '-I./build/include' ] diff --git a/CMakeLists.txt b/CMakeLists.txt index e9e42697a..448b32294 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -147,6 +147,9 @@ include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR}) include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) include_directories(SYSTEM ${FMT_INCLUDE_DIR}) include_directories(SYSTEM ${ANTLR4_INCLUDE_DIR}) +include_directories(SYSTEM ${BZIP2_INCLUDE_DIR}) +include_directories(SYSTEM ${ZLIB_INCLUDE_DIR}) +include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIR}) # ----------------------------------------------------------------------------- # openCypher parser ----------------------------------------------------------- diff --git a/docs/dev/toolchain-bootstrap.md b/docs/dev/toolchain-bootstrap.md index 149285844..c7a8255cc 100644 --- a/docs/dev/toolchain-bootstrap.md +++ b/docs/dev/toolchain-bootstrap.md @@ -120,6 +120,7 @@ Requirements on CentOS 7: * boost-static (too low version --- compile manually) * rpm-build (RPM) * python3 (tests, ...) + * which (required for rocksdb) ### Boost 1.62 diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index edee97344..9a4fe3a83 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -43,7 +43,8 @@ endfunction(import_library) function(add_external_project name) set(options NO_C_COMPILER) set(one_value_kwargs SOURCE_DIR) - set(multi_value_kwargs CMAKE_ARGS DEPENDS INSTALL_COMMAND) + set(multi_value_kwargs CMAKE_ARGS DEPENDS INSTALL_COMMAND BUILD_COMMAND + CONFIGURE_COMMAND) cmake_parse_arguments(KW "${options}" "${one_value_kwargs}" "${multi_value_kwargs}" ${ARGN}) set(source_dir ${CMAKE_CURRENT_SOURCE_DIR}/${name}) if (KW_SOURCE_DIR) @@ -54,11 +55,13 @@ function(add_external_project name) endif() ExternalProject_Add(${name}-proj DEPENDS ${KW_DEPENDS} PREFIX ${source_dir} SOURCE_DIR ${source_dir} + CONFIGURE_COMMAND ${KW_CONFIGURE_COMMAND} CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_INSTALL_PREFIX=${source_dir} ${KW_CMAKE_ARGS} - INSTALL_COMMAND ${KW_INSTALL_COMMAND}) + INSTALL_COMMAND ${KW_INSTALL_COMMAND} + BUILD_COMMAND ${KW_BUILD_COMMAND}) endfunction(add_external_project) # Calls `add_external_project`, sets NAME_LIBRARY, NAME_INCLUDE_DIR variables @@ -168,3 +171,33 @@ import_header_library(cppitertools ${CMAKE_CURRENT_SOURCE_DIR}) # Setup json import_header_library(json ${CMAKE_CURRENT_SOURCE_DIR}) + +# Setup bzip2 +import_external_library(bzip2 STATIC + ${CMAKE_CURRENT_SOURCE_DIR}/bzip2/libbz2.a + ${CMAKE_CURRENT_SOURCE_DIR}/bzip2 + # bzip2's Makefile has -g CFLAG which is redundant + CONFIGURE_COMMAND sed -i "s/-Wall -Winline -O2 -g/-Wall -Winline -O2/g" ${CMAKE_CURRENT_SOURCE_DIR}/bzip2/Makefile + BUILD_COMMAND make -C ${CMAKE_CURRENT_SOURCE_DIR}/bzip2 + CC=${CMAKE_C_COMPILER} + CXX=${CMAKE_CXX_COMPILER} + INSTALL_COMMAND true) + +# Setup zlib +import_external_library(zlib STATIC + ${CMAKE_CURRENT_SOURCE_DIR}/zlib/lib/libz.a + ${CMAKE_CURRENT_SOURCE_DIR}/zlib) + +# Setup RocksDB +import_external_library(rocksdb STATIC + ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/librocksdb.a + ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include + # RocksDB's cmake on Linux doesn't generate static_lib target. + # That's the reason why NoOps (true) are used as configure + # and install commands. Build command uses RocksDB's Makefile. + CONFIGURE_COMMAND true + BUILD_COMMAND make -C ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb static_lib + -j${NPROC} + CC=${CMAKE_C_COMPILER} + CXX=${CMAKE_CXX_COMPILER} + INSTALL_COMMAND true) diff --git a/libs/setup.sh b/libs/setup.sh index e1c5ef352..38c789b98 100755 --- a/libs/setup.sh +++ b/libs/setup.sh @@ -111,3 +111,12 @@ cd .. # git clone https://github.com/r-lyeh/ltalloc.git ltalloc_tag="43b51c14857111f993f277c46151fdfac91525a2" # Nov 16, 2017 clone git://deps.memgraph.io/ltalloc.git ltalloc $ltalloc_tag + +bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580" # v1.0.6 (May 26, 2011) +clone git://deps.memgraph.io/bzip2.git bzip2 $bzip2_tag + +zlib_tag="v1.2.11" +clone git://deps.memgraph.io/zlib.git zlib $zlib_tag + +rocksdb_tag="v5.11.3" # Mar 12, 2018 +clone git://deps.memgraph.io/rocksdb.git rocksdb $rocksdb_tag diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 371fcd7d4..9d6273f9a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -97,6 +97,10 @@ add_library(memgraph_lib STATIC ${memgraph_src_files}) target_link_libraries(memgraph_lib ${MEMGRAPH_ALL_LIBS}) add_dependencies(memgraph_lib generate_opencypher_parser) +# STATIC library used to store key-value pairs +add_library(kvstore_lib STATIC storage/kvstore.cpp) +target_link_libraries(kvstore_lib stdc++fs rocksdb bzip2 zlib) + # Generate a version.hpp file set(VERSION_STRING ${memgraph_VERSION}) configure_file(version.hpp.in version.hpp @ONLY) diff --git a/src/database/graph_db.cpp b/src/database/graph_db.cpp index 324dfbca2..f0f69aa55 100644 --- a/src/database/graph_db.cpp +++ b/src/database/graph_db.cpp @@ -34,6 +34,7 @@ #include "transactions/engine_master.hpp" #include "transactions/engine_single_node.hpp" #include "transactions/engine_worker.hpp" +#include "utils/file.hpp" #include "utils/flag_validation.hpp" using namespace storage; @@ -302,7 +303,7 @@ class Worker : public PrivateBase { PublicBase::PublicBase(std::unique_ptr<PrivateBase> impl) : impl_(std::move(impl)) { if (impl_->config_.durability_enabled) - durability::CheckDurabilityDir(impl_->config_.durability_directory); + utils::CheckDir(impl_->config_.durability_directory); // Durability recovery. { diff --git a/src/durability/paths.cpp b/src/durability/paths.cpp index 93f29d8c8..f773a9bd0 100644 --- a/src/durability/paths.cpp +++ b/src/durability/paths.cpp @@ -11,29 +11,9 @@ #include "utils/timestamp.hpp" namespace durability { + namespace fs = std::experimental::filesystem; -bool EnsureDir(const fs::path &dir) { - std::error_code error_code; // Just for exception suppression. - auto result = fs::create_directories(dir, error_code); - // The result will be false if the directory already exists. This is why we - // also check the error_code value. - return result || !error_code.value(); -} - -void CheckDurabilityDir(const std::string &durability_dir) { - namespace fs = std::experimental::filesystem; - if (fs::exists(durability_dir)) { - CHECK(fs::is_directory(durability_dir)) << "The durability directory path '" - << durability_dir - << "' is not a directory!"; - } else { - bool success = EnsureDir(durability_dir); - CHECK(success) << "Failed to create durability directory '" - << durability_dir << "'."; - } -} - std::experimental::optional<tx::TransactionId> TransactionIdFromWalFilename( const std::string &name) { auto nullopt = std::experimental::nullopt; diff --git a/src/durability/paths.hpp b/src/durability/paths.hpp index d4b38356d..ebb769d79 100644 --- a/src/durability/paths.hpp +++ b/src/durability/paths.hpp @@ -9,13 +9,6 @@ namespace durability { const std::string kSnapshotDir = "snapshots"; const std::string kWalDir = "wal"; -/// Esures that the given dir either exists or is succsefully created. -bool EnsureDir(const std::experimental::filesystem::path &dir); - -/// Ensures the given durability directory exists and is ready for use. Creates -/// the directory if it doesn't exist. -void CheckDurabilityDir(const std::string &durability_dir); - /// Returns the transaction id contained in the file name. If the filename is /// not a parseable WAL file name, nullopt is returned. If the filename /// represents the "current" WAL file, then the maximum possible transaction ID diff --git a/src/durability/snapshooter.cpp b/src/durability/snapshooter.cpp index 74d50af54..c74a273a5 100644 --- a/src/durability/snapshooter.cpp +++ b/src/durability/snapshooter.cpp @@ -9,6 +9,7 @@ #include "durability/paths.hpp" #include "durability/snapshot_encoder.hpp" #include "durability/version.hpp" +#include "utils/file.hpp" namespace fs = std::experimental::filesystem; @@ -122,7 +123,7 @@ void RemoveOldWals(const fs::path &wal_dir, bool MakeSnapshot(database::GraphDb &db, database::GraphDbAccessor &dba, const fs::path &durability_dir, const int snapshot_max_retained) { - if (!EnsureDir(durability_dir / kSnapshotDir)) return false; + if (!utils::EnsureDir(durability_dir / kSnapshotDir)) return false; const auto snapshot_file = MakeSnapshotPath(durability_dir, db.WorkerId(), dba.transaction_id()); if (fs::exists(snapshot_file)) return false; diff --git a/src/durability/wal.cpp b/src/durability/wal.cpp index de6b2d661..6ddc074e9 100644 --- a/src/durability/wal.cpp +++ b/src/durability/wal.cpp @@ -2,6 +2,7 @@ #include "communication/bolt/v1/decoder/decoded_value.hpp" #include "durability/paths.hpp" +#include "utils/file.hpp" #include "utils/flag_validation.hpp" DEFINE_HIDDEN_int32( @@ -24,7 +25,7 @@ WriteAheadLog::WriteAheadLog( bool durability_enabled) : deltas_{FLAGS_wal_buffer_size}, wal_file_{worker_id, durability_dir} { if (durability_enabled) { - CheckDurabilityDir(durability_dir); + utils::CheckDir(durability_dir); wal_file_.Init(); scheduler_.Run("WAL", std::chrono::milliseconds(FLAGS_wal_flush_interval_millis), @@ -47,7 +48,7 @@ WriteAheadLog::WalFile::~WalFile() { } void WriteAheadLog::WalFile::Init() { - if (!EnsureDir(wal_dir_)) { + if (!utils::EnsureDir(wal_dir_)) { LOG(ERROR) << "Can't write to WAL directory: " << wal_dir_; current_wal_file_ = std::experimental::filesystem::path(); } else { diff --git a/src/storage/kvstore.cpp b/src/storage/kvstore.cpp new file mode 100644 index 000000000..25b5835de --- /dev/null +++ b/src/storage/kvstore.cpp @@ -0,0 +1,41 @@ +#include "storage/kvstore.hpp" + +#include "durability/paths.hpp" +#include "utils/file.hpp" + +namespace storage { + +KVStore::KVStore(fs::path storage) : storage_(storage) { + if (!utils::EnsureDir(storage)) + throw KVStoreError("Folder for the key-value store " + storage.string() + + " couldn't be initialized!"); + options_.create_if_missing = true; + rocksdb::DB *db = nullptr; + auto s = rocksdb::DB::Open(options_, storage.c_str(), &db); + if (!s.ok()) + throw KVStoreError("RocksDB couldn't be initialized inside " + + storage.string() + "!"); + db_.reset(db); +} + +bool KVStore::Put(const std::string &key, const std::string &value) { + auto s = db_->Put(rocksdb::WriteOptions(), key.c_str(), value.c_str()); + if (!s.ok()) return false; + return true; +} + +std::experimental::optional<std::string> KVStore::Get( + const std::string &key) const noexcept { + std::string value; + auto s = db_->Get(rocksdb::ReadOptions(), key.c_str(), &value); + if (!s.ok()) return std::experimental::nullopt; + return value; +} + +bool KVStore::Delete(const std::string &key) { + auto s = db_->SingleDelete(rocksdb::WriteOptions(), key.c_str()); + if (!s.ok()) return false; + return true; +} + +} // namespace storage diff --git a/src/storage/kvstore.hpp b/src/storage/kvstore.hpp new file mode 100644 index 000000000..ae9997b6f --- /dev/null +++ b/src/storage/kvstore.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include <experimental/filesystem> +#include <experimental/optional> +#include <string> + +#include <rocksdb/db.h> +#include <rocksdb/options.h> + +#include "utils/exceptions.hpp" + +namespace storage { + +namespace fs = std::experimental::filesystem; + +class KVStoreError : public utils::BasicException { + public: + using utils::BasicException::BasicException; +}; + +/** + * Abstraction used to manage key-value pairs. The underlying implementation + * guarantees thread safety and durability properties. + */ +class KVStore final { + public: + /** + * @param storage Path to a directory where the data is persisted. + * + * NOTE: Don't instantiate more instances of a KVStore with the same + * storage directory because that will lead to undefined behaviour. + */ + explicit KVStore(fs::path storage); + + /** + * Store value under the given key. + * + * @param key + * @param value + * + * @return true if the value has been successfully stored. + * In case of any error false is going to be returned. + */ + bool Put(const std::string &key, const std::string &value); + + /** + * Retrieve value for the given key. + * + * @param key + * + * @return Value for the given key. std::nullopt in case of any error + * OR the value doesn't exist. + */ + std::experimental::optional<std::string> Get(const std::string &key) const + noexcept; + + /** + * Delete value under the given key. + * + * @param key + * + * @return True on success, false on error. The return value is + * true if the key doesn't exist and underlying storage + * didn't encounter any error. + */ + bool Delete(const std::string &key); + + private: + fs::path storage_; + std::unique_ptr<rocksdb::DB> db_; + rocksdb::Options options_; +}; + +} // namespace storage diff --git a/src/utils/file.cpp b/src/utils/file.cpp index 820c99915..dffd30f7f 100644 --- a/src/utils/file.cpp +++ b/src/utils/file.cpp @@ -63,6 +63,22 @@ void Write(const std::string &text, const fs::path &path) { stream.close(); } +bool EnsureDir(const fs::path &dir) { + if (fs::exists(dir)) return true; + std::error_code error_code; // Just for exception suppression. + return fs::create_directories(dir, error_code); +} + +void CheckDir(const std::string &dir) { + if (fs::exists(dir)) { + CHECK(fs::is_directory(dir)) << "The directory path '" << dir + << "' is not a directory!"; + } else { + bool success = EnsureDir(dir); + CHECK(success) << "Failed to create directory '" << dir << "'."; + } +} + File::File() : fd_(-1), path_() {} File::File(int fd, fs::path path) : fd_(fd), path_(std::move(path)) {} diff --git a/src/utils/file.hpp b/src/utils/file.hpp index 1ebac821d..7751a30f5 100644 --- a/src/utils/file.hpp +++ b/src/utils/file.hpp @@ -48,6 +48,17 @@ std::vector<std::string> ReadLines(const fs::path &path); */ void Write(const std::string &text, const fs::path &path); +/** + * Esures that the given dir either exists or is succsefully created. + */ +bool EnsureDir(const std::experimental::filesystem::path &dir); + +/** + * Ensures the given directory exists and is ready for use. Creates + * the directory if it doesn't exist. + */ +void CheckDir(const std::string &dir); + // End higher level operations. // Lower level wrappers around C system calls follow. diff --git a/tests/benchmark/expansion.cpp b/tests/benchmark/expansion.cpp index ce2b3232e..ee44f452a 100644 --- a/tests/benchmark/expansion.cpp +++ b/tests/benchmark/expansion.cpp @@ -72,7 +72,6 @@ BENCHMARK_REGISTER_F(ExpansionBenchFixture, Expand) ->Unit(benchmark::kMillisecond); int main(int argc, char **argv) { - gflags::ParseCommandLineFlags(&argc, &argv, true); google::InitGoogleLogging(argv[0]); ::benchmark::Initialize(&argc, argv); diff --git a/tests/benchmark/mvcc.cpp b/tests/benchmark/mvcc.cpp index 03d0de517..0bfffa23f 100644 --- a/tests/benchmark/mvcc.cpp +++ b/tests/benchmark/mvcc.cpp @@ -60,7 +60,6 @@ BENCHMARK(MvccMix) ->Range(1 << 14, 1 << 23) // 1<<14, 1<<15, 1<<16, ... ->Unit(benchmark::kMillisecond); -DEFINE_string(hehehe, "bok", "ne"); int main(int argc, char **argv) { google::InitGoogleLogging(argv[0]); diff --git a/tests/manual/CMakeLists.txt b/tests/manual/CMakeLists.txt index d830bbada..c2efc3c6a 100644 --- a/tests/manual/CMakeLists.txt +++ b/tests/manual/CMakeLists.txt @@ -22,8 +22,6 @@ foreach(test_cpp ${test_type_cpps}) set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name}) # link libraries - target_link_libraries(${target_name} memgraph_lib) - # gtest - target_link_libraries(${target_name} gtest gtest_main) - + target_link_libraries(${target_name} memgraph_lib gtest gtest_main + glog kvstore_lib) endforeach() diff --git a/tests/manual/kvstore.cpp b/tests/manual/kvstore.cpp new file mode 100644 index 000000000..f8c2e7e46 --- /dev/null +++ b/tests/manual/kvstore.cpp @@ -0,0 +1,48 @@ +#include <glog/logging.h> +#include <gtest/gtest.h> + +#include "durability/paths.hpp" +#include "storage/kvstore.hpp" +#include "utils/file.hpp" + +namespace fs = std::experimental::filesystem; + +class KVStore : public ::testing::Test { + protected: + virtual void SetUp() { utils::EnsureDir(test_folder_); } + + virtual void TearDown() { fs::remove_all(test_folder_); } + + fs::path test_folder_{fs::path("kvstore_test")}; +}; + +TEST_F(KVStore, PutGet) { + storage::KVStore kvstore(test_folder_ / "PutGet"); + ASSERT_TRUE(kvstore.Put("key", "value")); + ASSERT_EQ(kvstore.Get("key").value(), "value"); +} + +TEST_F(KVStore, PutGetDeleteGet) { + storage::KVStore kvstore(test_folder_ / "PutGetDeleteGet"); + ASSERT_TRUE(kvstore.Put("key", "value")); + ASSERT_EQ(kvstore.Get("key").value(), "value"); + ASSERT_TRUE(kvstore.Delete("key")); + ASSERT_FALSE(static_cast<bool>(kvstore.Get("key"))); +} + +TEST_F(KVStore, Durability) { + { + storage::KVStore kvstore(test_folder_ / "Durability"); + ASSERT_TRUE(kvstore.Put("key", "value")); + } + { + storage::KVStore kvstore(test_folder_ / "Durability"); + ASSERT_EQ(kvstore.Get("key").value(), "value"); + } +} + +int main(int argc, char **argv) { + google::InitGoogleLogging(argv[0]); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/manual/snapshot_generation/snapshot_writer.hpp b/tests/manual/snapshot_generation/snapshot_writer.hpp index 6ef111e00..2833f3a99 100644 --- a/tests/manual/snapshot_generation/snapshot_writer.hpp +++ b/tests/manual/snapshot_generation/snapshot_writer.hpp @@ -7,6 +7,7 @@ #include "durability/paths.hpp" #include "durability/version.hpp" #include "query/typed_value.hpp" +#include "utils/file.hpp" #include "graph_state.hpp" @@ -116,7 +117,7 @@ void WriteToSnapshot(GraphState &state, const std::string &path) { const std::experimental::filesystem::path durability_dir = path / std::experimental::filesystem::path("worker_" + std::to_string(worker_id)); - if (!durability::EnsureDir(durability_dir / "snapshots")) { + if (!utils::EnsureDir(durability_dir / "snapshots")) { LOG(ERROR) << "Unable to create durability directory!"; exit(0); }