Add rocksdb as a lib + initial implementation of the KVStore

Reviewers: teon.banek, mferencevic, ipaljak

Reviewed By: ipaljak

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1331

.ycm_extra_conf.py

@@ -32,6 +32,9 @@ BASE_FLAGS = [
     '-I./libs/gflags/include',
     '-I./experimental/distributed/src',
     '-I./libs/postgresql/include',
+    '-I./libs/bzip2',
+    '-I./libs/zlib',
+    '-I./libs/rocksdb/include',
     '-I./build/include'
 ]
 

CMakeLists.txt

@@ -147,6 +147,9 @@ include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR})
 include_directories(SYSTEM ${GLOG_INCLUDE_DIR})
 include_directories(SYSTEM ${FMT_INCLUDE_DIR})
 include_directories(SYSTEM ${ANTLR4_INCLUDE_DIR})
+include_directories(SYSTEM ${BZIP2_INCLUDE_DIR})
+include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
+include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIR})
 # -----------------------------------------------------------------------------
 
 # openCypher parser -----------------------------------------------------------

docs/dev/toolchain-bootstrap.md

@@ -120,6 +120,7 @@ Requirements on CentOS 7:
   * boost-static (too low version --- compile manually)
   * rpm-build (RPM)
   * python3 (tests, ...)
+  * which (required for rocksdb)
 
 ### Boost 1.62
 

libs/CMakeLists.txt

@@ -43,7 +43,8 @@ endfunction(import_library)
 function(add_external_project name)
   set(options NO_C_COMPILER)
   set(one_value_kwargs SOURCE_DIR)
-  set(multi_value_kwargs CMAKE_ARGS DEPENDS INSTALL_COMMAND)
+  set(multi_value_kwargs CMAKE_ARGS DEPENDS INSTALL_COMMAND BUILD_COMMAND
+      CONFIGURE_COMMAND)
   cmake_parse_arguments(KW "${options}" "${one_value_kwargs}" "${multi_value_kwargs}" ${ARGN})
   set(source_dir ${CMAKE_CURRENT_SOURCE_DIR}/${name})
   if (KW_SOURCE_DIR)
@@ -54,11 +55,13 @@ function(add_external_project name)
   endif()
   ExternalProject_Add(${name}-proj DEPENDS ${KW_DEPENDS}
                       PREFIX ${source_dir} SOURCE_DIR ${source_dir}
+                      CONFIGURE_COMMAND ${KW_CONFIGURE_COMMAND}
                       CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release
                                  -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
                                  -DCMAKE_INSTALL_PREFIX=${source_dir}
                                  ${KW_CMAKE_ARGS}
-                      INSTALL_COMMAND ${KW_INSTALL_COMMAND})
+                      INSTALL_COMMAND ${KW_INSTALL_COMMAND}
+                      BUILD_COMMAND ${KW_BUILD_COMMAND})
 endfunction(add_external_project)
 
 # Calls `add_external_project`, sets NAME_LIBRARY, NAME_INCLUDE_DIR variables
@@ -168,3 +171,33 @@ import_header_library(cppitertools ${CMAKE_CURRENT_SOURCE_DIR})
 
 # Setup json
 import_header_library(json ${CMAKE_CURRENT_SOURCE_DIR})
+
+# Setup bzip2
+import_external_library(bzip2 STATIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/bzip2/libbz2.a
+    ${CMAKE_CURRENT_SOURCE_DIR}/bzip2
+    # bzip2's Makefile has -g CFLAG which is redundant
+    CONFIGURE_COMMAND sed -i "s/-Wall -Winline -O2 -g/-Wall -Winline -O2/g" ${CMAKE_CURRENT_SOURCE_DIR}/bzip2/Makefile
+    BUILD_COMMAND make -C ${CMAKE_CURRENT_SOURCE_DIR}/bzip2
+                       CC=${CMAKE_C_COMPILER}
+                       CXX=${CMAKE_CXX_COMPILER}
+    INSTALL_COMMAND true)
+
+# Setup zlib
+import_external_library(zlib STATIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/zlib/lib/libz.a
+    ${CMAKE_CURRENT_SOURCE_DIR}/zlib)
+
+# Setup RocksDB
+import_external_library(rocksdb STATIC
+  ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/librocksdb.a
+  ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include
+  # RocksDB's cmake on Linux doesn't generate static_lib target.
+  # That's the reason why NoOps (true) are used as configure
+  # and install commands. Build command uses RocksDB's Makefile.
+  CONFIGURE_COMMAND true
+  BUILD_COMMAND make -C ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb static_lib
+                     -j${NPROC}
+                     CC=${CMAKE_C_COMPILER}
+                     CXX=${CMAKE_CXX_COMPILER}
+  INSTALL_COMMAND true)

libs/setup.sh

@@ -111,3 +111,12 @@ cd ..
 # git clone https://github.com/r-lyeh/ltalloc.git
 ltalloc_tag="43b51c14857111f993f277c46151fdfac91525a2" # Nov 16, 2017
 clone git://deps.memgraph.io/ltalloc.git ltalloc $ltalloc_tag
+
+bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580"  # v1.0.6 (May 26, 2011)
+clone git://deps.memgraph.io/bzip2.git bzip2 $bzip2_tag
+
+zlib_tag="v1.2.11"
+clone git://deps.memgraph.io/zlib.git zlib $zlib_tag
+
+rocksdb_tag="v5.11.3" # Mar 12, 2018
+clone git://deps.memgraph.io/rocksdb.git rocksdb $rocksdb_tag

src/CMakeLists.txt

@@ -97,6 +97,10 @@ add_library(memgraph_lib STATIC ${memgraph_src_files})
 target_link_libraries(memgraph_lib ${MEMGRAPH_ALL_LIBS})
 add_dependencies(memgraph_lib generate_opencypher_parser)
 
+# STATIC library used to store key-value pairs
+add_library(kvstore_lib STATIC storage/kvstore.cpp)
+target_link_libraries(kvstore_lib stdc++fs rocksdb bzip2 zlib)
+
 # Generate a version.hpp file
 set(VERSION_STRING ${memgraph_VERSION})
 configure_file(version.hpp.in version.hpp @ONLY)

src/database/graph_db.cpp

@@ -34,6 +34,7 @@
 #include "transactions/engine_master.hpp"
 #include "transactions/engine_single_node.hpp"
 #include "transactions/engine_worker.hpp"
+#include "utils/file.hpp"
 #include "utils/flag_validation.hpp"
 
 using namespace storage;
@@ -302,7 +303,7 @@ class Worker : public PrivateBase {
 PublicBase::PublicBase(std::unique_ptr<PrivateBase> impl)
     : impl_(std::move(impl)) {
   if (impl_->config_.durability_enabled)
-    durability::CheckDurabilityDir(impl_->config_.durability_directory);
+    utils::CheckDir(impl_->config_.durability_directory);
 
   // Durability recovery.
   {

src/durability/paths.cpp

@@ -11,29 +11,9 @@
 #include "utils/timestamp.hpp"
 
 namespace durability {
+
 namespace fs = std::experimental::filesystem;
 
-bool EnsureDir(const fs::path &dir) {
-  std::error_code error_code;  // Just for exception suppression.
-  auto result = fs::create_directories(dir, error_code);
-  // The result will be false if the directory already exists. This is why we
-  // also check the error_code value.
-  return result || !error_code.value();
-}
-
-void CheckDurabilityDir(const std::string &durability_dir) {
-  namespace fs = std::experimental::filesystem;
-  if (fs::exists(durability_dir)) {
-    CHECK(fs::is_directory(durability_dir)) << "The durability directory path '"
-                                            << durability_dir
-                                            << "' is not a directory!";
-  } else {
-    bool success = EnsureDir(durability_dir);
-    CHECK(success) << "Failed to create durability directory '"
-                   << durability_dir << "'.";
-  }
-}
-
 std::experimental::optional<tx::TransactionId> TransactionIdFromWalFilename(
     const std::string &name) {
   auto nullopt = std::experimental::nullopt;

src/durability/paths.hpp

@@ -9,13 +9,6 @@ namespace durability {
 const std::string kSnapshotDir = "snapshots";
 const std::string kWalDir = "wal";
 
-/// Esures that the given dir either exists or is succsefully created.
-bool EnsureDir(const std::experimental::filesystem::path &dir);
-
-/// Ensures the given durability directory exists and is ready for use. Creates
-/// the directory if it doesn't exist.
-void CheckDurabilityDir(const std::string &durability_dir);
-
 /// Returns the transaction id contained in the file name. If the filename is
 /// not a parseable WAL file name, nullopt is returned. If the filename
 /// represents the "current" WAL file, then the maximum possible transaction ID

src/durability/snapshooter.cpp

@@ -9,6 +9,7 @@
 #include "durability/paths.hpp"
 #include "durability/snapshot_encoder.hpp"
 #include "durability/version.hpp"
+#include "utils/file.hpp"
 
 namespace fs = std::experimental::filesystem;
 
@@ -122,7 +123,7 @@ void RemoveOldWals(const fs::path &wal_dir,
 bool MakeSnapshot(database::GraphDb &db, database::GraphDbAccessor &dba,
                   const fs::path &durability_dir,
                   const int snapshot_max_retained) {
-  if (!EnsureDir(durability_dir / kSnapshotDir)) return false;
+  if (!utils::EnsureDir(durability_dir / kSnapshotDir)) return false;
   const auto snapshot_file =
       MakeSnapshotPath(durability_dir, db.WorkerId(), dba.transaction_id());
   if (fs::exists(snapshot_file)) return false;

src/durability/wal.cpp

@@ -2,6 +2,7 @@
 
 #include "communication/bolt/v1/decoder/decoded_value.hpp"
 #include "durability/paths.hpp"
+#include "utils/file.hpp"
 #include "utils/flag_validation.hpp"
 
 DEFINE_HIDDEN_int32(
@@ -24,7 +25,7 @@ WriteAheadLog::WriteAheadLog(
     bool durability_enabled)
     : deltas_{FLAGS_wal_buffer_size}, wal_file_{worker_id, durability_dir} {
   if (durability_enabled) {
-    CheckDurabilityDir(durability_dir);
+    utils::CheckDir(durability_dir);
     wal_file_.Init();
     scheduler_.Run("WAL",
                    std::chrono::milliseconds(FLAGS_wal_flush_interval_millis),
@@ -47,7 +48,7 @@ WriteAheadLog::WalFile::~WalFile() {
 }
 
 void WriteAheadLog::WalFile::Init() {
-  if (!EnsureDir(wal_dir_)) {
+  if (!utils::EnsureDir(wal_dir_)) {
     LOG(ERROR) << "Can't write to WAL directory: " << wal_dir_;
     current_wal_file_ = std::experimental::filesystem::path();
   } else {

src/storage/kvstore.cpp

@@ -0,0 +1,41 @@
+#include "storage/kvstore.hpp"
+
+#include "durability/paths.hpp"
+#include "utils/file.hpp"
+
+namespace storage {
+
+KVStore::KVStore(fs::path storage) : storage_(storage) {
+  if (!utils::EnsureDir(storage))
+    throw KVStoreError("Folder for the key-value store " + storage.string() +
+                       " couldn't be initialized!");
+  options_.create_if_missing = true;
+  rocksdb::DB *db = nullptr;
+  auto s = rocksdb::DB::Open(options_, storage.c_str(), &db);
+  if (!s.ok())
+    throw KVStoreError("RocksDB couldn't be initialized inside " +
+                       storage.string() + "!");
+  db_.reset(db);
+}
+
+bool KVStore::Put(const std::string &key, const std::string &value) {
+  auto s = db_->Put(rocksdb::WriteOptions(), key.c_str(), value.c_str());
+  if (!s.ok()) return false;
+  return true;
+}
+
+std::experimental::optional<std::string> KVStore::Get(
+    const std::string &key) const noexcept {
+  std::string value;
+  auto s = db_->Get(rocksdb::ReadOptions(), key.c_str(), &value);
+  if (!s.ok()) return std::experimental::nullopt;
+  return value;
+}
+
+bool KVStore::Delete(const std::string &key) {
+  auto s = db_->SingleDelete(rocksdb::WriteOptions(), key.c_str());
+  if (!s.ok()) return false;
+  return true;
+}
+
+}  // namespace storage

src/storage/kvstore.hpp

@@ -0,0 +1,74 @@
+#pragma once
+
+#include <experimental/filesystem>
+#include <experimental/optional>
+#include <string>
+
+#include <rocksdb/db.h>
+#include <rocksdb/options.h>
+
+#include "utils/exceptions.hpp"
+
+namespace storage {
+
+namespace fs = std::experimental::filesystem;
+
+class KVStoreError : public utils::BasicException {
+ public:
+  using utils::BasicException::BasicException;
+};
+
+/**
+ * Abstraction used to manage key-value pairs. The underlying implementation
+ * guarantees thread safety and durability properties.
+ */
+class KVStore final {
+ public:
+  /**
+   * @param storage Path to a directory where the data is persisted.
+   *
+   * NOTE: Don't instantiate more instances of a KVStore with the same
+   *       storage directory because that will lead to undefined behaviour.
+   */
+  explicit KVStore(fs::path storage);
+
+  /**
+   * Store value under the given key.
+   *
+   * @param key
+   * @param value
+   *
+   * @return true if the value has been successfully stored.
+   *         In case of any error false is going to be returned.
+   */
+  bool Put(const std::string &key, const std::string &value);
+
+  /**
+   * Retrieve value for the given key.
+   *
+   * @param key
+   *
+   * @return Value for the given key. std::nullopt in case of any error
+   *         OR the value doesn't exist.
+   */
+  std::experimental::optional<std::string> Get(const std::string &key) const
+      noexcept;
+
+  /**
+   * Delete value under the given key.
+   *
+   * @param key
+   *
+   * @return True on success, false on error. The return value is
+   *         true if the key doesn't exist and underlying storage
+   *         didn't encounter any error.
+   */
+  bool Delete(const std::string &key);
+
+ private:
+  fs::path storage_;
+  std::unique_ptr<rocksdb::DB> db_;
+  rocksdb::Options options_;
+};
+
+}  // namespace storage

src/utils/file.cpp

@@ -63,6 +63,22 @@ void Write(const std::string &text, const fs::path &path) {
   stream.close();
 }
 
+bool EnsureDir(const fs::path &dir) {
+  if (fs::exists(dir)) return true;
+  std::error_code error_code;  // Just for exception suppression.
+  return fs::create_directories(dir, error_code);
+}
+
+void CheckDir(const std::string &dir) {
+  if (fs::exists(dir)) {
+    CHECK(fs::is_directory(dir)) << "The directory path '" << dir
+                                 << "' is not a directory!";
+  } else {
+    bool success = EnsureDir(dir);
+    CHECK(success) << "Failed to create directory '" << dir << "'.";
+  }
+}
+
 File::File() : fd_(-1), path_() {}
 
 File::File(int fd, fs::path path) : fd_(fd), path_(std::move(path)) {}

src/utils/file.hpp

@@ -48,6 +48,17 @@ std::vector<std::string> ReadLines(const fs::path &path);
  */
 void Write(const std::string &text, const fs::path &path);
 
+/**
+ * Esures that the given dir either exists or is succsefully created.
+ */
+bool EnsureDir(const std::experimental::filesystem::path &dir);
+
+/**
+ * Ensures the given directory exists and is ready for use. Creates
+ * the directory if it doesn't exist.
+ */
+void CheckDir(const std::string &dir);
+
 // End higher level operations.
 
 // Lower level wrappers around C system calls follow.

tests/benchmark/expansion.cpp

@@ -72,7 +72,6 @@ BENCHMARK_REGISTER_F(ExpansionBenchFixture, Expand)
     ->Unit(benchmark::kMillisecond);
 
 int main(int argc, char **argv) {
-  gflags::ParseCommandLineFlags(&argc, &argv, true);
   google::InitGoogleLogging(argv[0]);
 
   ::benchmark::Initialize(&argc, argv);

tests/benchmark/mvcc.cpp

@@ -60,7 +60,6 @@ BENCHMARK(MvccMix)
     ->Range(1 << 14, 1 << 23)  // 1<<14, 1<<15, 1<<16, ...
     ->Unit(benchmark::kMillisecond);
 
-DEFINE_string(hehehe, "bok", "ne");
 int main(int argc, char **argv) {
   google::InitGoogleLogging(argv[0]);
 

tests/manual/CMakeLists.txt

@@ -22,8 +22,6 @@ foreach(test_cpp ${test_type_cpps})
     set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name})
 
     # link libraries
-    target_link_libraries(${target_name} memgraph_lib)
-    # gtest
-    target_link_libraries(${target_name} gtest gtest_main)
-
+    target_link_libraries(${target_name} memgraph_lib gtest gtest_main
+        glog kvstore_lib)
 endforeach()

tests/manual/kvstore.cpp

@@ -0,0 +1,48 @@
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "durability/paths.hpp"
+#include "storage/kvstore.hpp"
+#include "utils/file.hpp"
+
+namespace fs = std::experimental::filesystem;
+
+class KVStore : public ::testing::Test {
+ protected:
+  virtual void SetUp() { utils::EnsureDir(test_folder_); }
+
+  virtual void TearDown() { fs::remove_all(test_folder_); }
+
+  fs::path test_folder_{fs::path("kvstore_test")};
+};
+
+TEST_F(KVStore, PutGet) {
+  storage::KVStore kvstore(test_folder_ / "PutGet");
+  ASSERT_TRUE(kvstore.Put("key", "value"));
+  ASSERT_EQ(kvstore.Get("key").value(), "value");
+}
+
+TEST_F(KVStore, PutGetDeleteGet) {
+  storage::KVStore kvstore(test_folder_ / "PutGetDeleteGet");
+  ASSERT_TRUE(kvstore.Put("key", "value"));
+  ASSERT_EQ(kvstore.Get("key").value(), "value");
+  ASSERT_TRUE(kvstore.Delete("key"));
+  ASSERT_FALSE(static_cast<bool>(kvstore.Get("key")));
+}
+
+TEST_F(KVStore, Durability) {
+  {
+    storage::KVStore kvstore(test_folder_ / "Durability");
+    ASSERT_TRUE(kvstore.Put("key", "value"));
+  }
+  {
+    storage::KVStore kvstore(test_folder_ / "Durability");
+    ASSERT_EQ(kvstore.Get("key").value(), "value");
+  }
+}
+
+int main(int argc, char **argv) {
+  google::InitGoogleLogging(argv[0]);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

tests/manual/snapshot_generation/snapshot_writer.hpp

@@ -7,6 +7,7 @@
 #include "durability/paths.hpp"
 #include "durability/version.hpp"
 #include "query/typed_value.hpp"
+#include "utils/file.hpp"
 
 #include "graph_state.hpp"
 
@@ -116,7 +117,7 @@ void WriteToSnapshot(GraphState &state, const std::string &path) {
     const std::experimental::filesystem::path durability_dir =
         path / std::experimental::filesystem::path("worker_" +
                                                    std::to_string(worker_id));
-    if (!durability::EnsureDir(durability_dir / "snapshots")) {
+    if (!utils::EnsureDir(durability_dir / "snapshots")) {
       LOG(ERROR) << "Unable to create durability directory!";
       exit(0);
     }