From 2924746f20d7302ac62acbdec10f5bfdf8807ffe Mon Sep 17 00:00:00 2001 From: Matej Ferencevic Date: Wed, 28 Mar 2018 14:49:28 +0200 Subject: [PATCH] Remove PostgreSQL Reviewers: buda, teon.banek Reviewed By: buda Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D1326 --- libs/setup.sh | 7 -- tests/distributed/card_fraud/apollo_runs.py | 1 - tests/macro_benchmark/CMakeLists.txt | 6 +- tests/macro_benchmark/apollo_runs.yaml | 1 - .../clients/card_fraud_client.cpp | 15 ++- tests/macro_benchmark/clients/common.hpp | 13 +-- .../clients/long_running_common.hpp | 15 ++- .../macro_benchmark/clients/pokec_client.cpp | 23 ++-- .../clients/postgres_client.hpp | 108 ------------------ .../macro_benchmark/clients/query_client.cpp | 54 ++------- tests/macro_benchmark/databases.py | 45 -------- 11 files changed, 46 insertions(+), 242 deletions(-) delete mode 100644 tests/macro_benchmark/clients/postgres_client.hpp diff --git a/libs/setup.sh b/libs/setup.sh index b1f59a6e6..e1c5ef352 100755 --- a/libs/setup.sh +++ b/libs/setup.sh @@ -98,13 +98,6 @@ rm -rf neo4j mv neo4j-community-3.2.3 neo4j rm neo4j.tar.gz -# postgresql -wget -nv http://deps.memgraph.io/postgresql-9.6.5-1-linux-x64-binaries.tar.gz -O postgres.tar.gz -tar -xzf postgres.tar.gz -rm -rf postgresql -mv pgsql postgresql -rm postgres.tar.gz - # nlohmann json # We wget header instead of cloning repo since repo is huge (lots of test data). # We use head on Sep 1, 2017 instead of last release since it was long time ago. diff --git a/tests/distributed/card_fraud/apollo_runs.py b/tests/distributed/card_fraud/apollo_runs.py index 557ab9025..60c8aa543 100755 --- a/tests/distributed/card_fraud/apollo_runs.py +++ b/tests/distributed/card_fraud/apollo_runs.py @@ -34,7 +34,6 @@ for i in range(NUM_MACHINES): "jail_service.py", "card_fraud/card_fraud.py", "card_fraud/snapshots/worker_" + str(i), - "../../libs/postgresql/lib", ] + additional, "outfile_paths": outfile_paths, "parallel_run": "distributed__card_fraud", diff --git a/tests/macro_benchmark/CMakeLists.txt b/tests/macro_benchmark/CMakeLists.txt index 202bf0140..6e09f79a8 100644 --- a/tests/macro_benchmark/CMakeLists.txt +++ b/tests/macro_benchmark/CMakeLists.txt @@ -5,10 +5,6 @@ get_filename_component(test_type ${CMAKE_CURRENT_SOURCE_DIR} NAME) file(GLOB_RECURSE test_type_cpps *.cpp) message(STATUS "Available ${test_type} cpp files are: ${test_type_cpps}") -add_library(postgres SHARED IMPORTED) -set_property(TARGET postgres PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${libs_dir}/postgresql/include) -set_property(TARGET postgres PROPERTY IMPORTED_LOCATION ${libs_dir}/postgresql/lib/libpq.so) - # add target that depends on all other targets set(all_targets_target memgraph__${test_type}) add_custom_target(${all_targets_target}) @@ -30,7 +26,7 @@ foreach(test_cpp ${test_type_cpps}) set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name}) # link libraries - target_link_libraries(${target_name} memgraph_lib postgres) + target_link_libraries(${target_name} memgraph_lib) # add target to dependencies add_dependencies(${all_targets_target} ${target_name}) diff --git a/tests/macro_benchmark/apollo_runs.yaml b/tests/macro_benchmark/apollo_runs.yaml index 0e75060c2..d8c6e4629 100644 --- a/tests/macro_benchmark/apollo_runs.yaml +++ b/tests/macro_benchmark/apollo_runs.yaml @@ -9,7 +9,6 @@ - ../../build_release/memgraph # memgraph release binary - ../../config # directory with config files - ../../build_release/tests/macro_benchmark # macro benchmark client binaries - - ../../libs/postgresql/lib # postgresql libs dir (for client binaries) outfile_paths: &MACRO_BENCHMARK_OUTFILE_PATHS - \./memgraph/tests/macro_benchmark/\.harness_summary diff --git a/tests/macro_benchmark/clients/card_fraud_client.cpp b/tests/macro_benchmark/clients/card_fraud_client.cpp index a823fda2a..7971a1def 100644 --- a/tests/macro_benchmark/clients/card_fraud_client.cpp +++ b/tests/macro_benchmark/clients/card_fraud_client.cpp @@ -5,11 +5,12 @@ #include "gflags/gflags.h" -#include "long_running_common.hpp" #include "stats/stats.hpp" #include "stats/stats_rpc_messages.hpp" #include "threading/sync/rwlock.hpp" +#include "long_running_common.hpp" + // TODO(mtomic): this sucks but I don't know a different way to make it work #include "boost/archive/binary_iarchive.hpp" #include "boost/archive/binary_oarchive.hpp" @@ -38,19 +39,19 @@ void UpdateStats() { num_edges.Set(2 * num_transactions); } -int64_t NumNodesWithLabel(BoltClient &client, std::string label) { +int64_t NumNodesWithLabel(Client &client, std::string label) { std::string query = fmt::format("MATCH (u :{}) RETURN count(u)", label); auto result = ExecuteNTimesTillSuccess(client, query, {}, MAX_RETRIES); return result.first.records[0][0].ValueInt(); } -int64_t MaxIdForLabel(BoltClient &client, std::string label) { +int64_t MaxIdForLabel(Client &client, std::string label) { std::string query = fmt::format("MATCH (u :{}) RETURN max(u.id)", label); auto result = ExecuteNTimesTillSuccess(client, query, {}, MAX_RETRIES); return result.first.records[0][0].ValueInt(); } -void CreateIndex(BoltClient &client, const std::string &label, +void CreateIndex(Client &client, const std::string &label, const std::string &property) { LOG(INFO) << fmt::format("Creating indexes for :{}({})...", label, property); ExecuteNTimesTillSuccess( @@ -342,7 +343,11 @@ int main(int argc, char **argv) { stats::InitStatsLogging( fmt::format("client.long_running.{}.{}", FLAGS_group, FLAGS_scenario)); - BoltClient client(FLAGS_address, FLAGS_port, FLAGS_username, FLAGS_password); + Endpoint endpoint(FLAGS_address, FLAGS_port); + Client client; + if (!client.Connect(endpoint, FLAGS_username, FLAGS_password)) { + LOG(FATAL) << "Couldn't connect to " << endpoint; + } num_pos.store(NumNodesWithLabel(client, "Pos")); num_cards.store(NumNodesWithLabel(client, "Card")); diff --git a/tests/macro_benchmark/clients/common.hpp b/tests/macro_benchmark/clients/common.hpp index feaa6b88f..a932f7566 100644 --- a/tests/macro_benchmark/clients/common.hpp +++ b/tests/macro_benchmark/clients/common.hpp @@ -11,11 +11,11 @@ #include "utils/exceptions.hpp" #include "utils/timer.hpp" -namespace { +using communication::bolt::Client; +using communication::bolt::DecodedValue; +using io::network::Endpoint; -void PrintJsonDecodedValue(std::ostream &os, - const communication::bolt::DecodedValue &value) { - using communication::bolt::DecodedValue; +void PrintJsonDecodedValue(std::ostream &os, const DecodedValue &value) { switch (value.type()) { case DecodedValue::Type::Null: os << "null"; @@ -55,9 +55,8 @@ void PrintJsonDecodedValue(std::ostream &os, } } -template std::pair ExecuteNTimesTillSuccess( - TClient &client, const std::string &query, + Client &client, const std::string &query, const std::map ¶ms, int max_attempts) { static thread_local std::mt19937 pseudo_rand_gen_{std::random_device{}()}; @@ -81,5 +80,3 @@ std::pair ExecuteNTimesTillSuccess( } } } - -} // namespace diff --git a/tests/macro_benchmark/clients/long_running_common.hpp b/tests/macro_benchmark/clients/long_running_common.hpp index 43f052293..f110a413a 100644 --- a/tests/macro_benchmark/clients/long_running_common.hpp +++ b/tests/macro_benchmark/clients/long_running_common.hpp @@ -2,14 +2,13 @@ #include "json/json.hpp" -#include "bolt_client.hpp" -#include "common.hpp" -#include "communication/bolt/v1/decoder/decoded_value.hpp" #include "stats/metrics.hpp" #include "stats/stats.hpp" #include "utils/network.hpp" #include "utils/timer.hpp" +#include "common.hpp" + const int MAX_RETRIES = 30; DEFINE_string(address, "127.0.0.1", "Server address"); @@ -28,8 +27,12 @@ auto &serialization_errors = stats::GetCounter("serialization_errors"); class TestClient { public: - TestClient() - : client_(FLAGS_address, FLAGS_port, FLAGS_username, FLAGS_password) {} + TestClient() { + Endpoint endpoint(FLAGS_address, FLAGS_port); + if (!client_.Connect(endpoint, FLAGS_username, FLAGS_password)) { + LOG(FATAL) << "Couldn't connect to " << endpoint; + } + } virtual ~TestClient() {} @@ -95,7 +98,7 @@ class TestClient { std::thread runner_thread_; private: - BoltClient client_; + Client client_; }; void RunMultithreadedTest(std::vector> &clients) { diff --git a/tests/macro_benchmark/clients/pokec_client.cpp b/tests/macro_benchmark/clients/pokec_client.cpp index 4ae4b81e0..3930deb89 100644 --- a/tests/macro_benchmark/clients/pokec_client.cpp +++ b/tests/macro_benchmark/clients/pokec_client.cpp @@ -13,18 +13,13 @@ #include #include -#include "bolt_client.hpp" -#include "common.hpp" -#include "communication/bolt/client.hpp" -#include "communication/bolt/v1/decoder/decoded_value.hpp" -#include "io/network/endpoint.hpp" -#include "io/network/socket.hpp" -#include "long_running_common.hpp" #include "threading/sync/spinlock.hpp" #include "utils/algorithm.hpp" #include "utils/network.hpp" #include "utils/timer.hpp" +#include "long_running_common.hpp" + using communication::bolt::DecodedEdge; using communication::bolt::DecodedValue; using communication::bolt::DecodedVertex; @@ -223,14 +218,14 @@ class PokecClient : public TestClient { } }; -int64_t NumNodes(BoltClient &client, const std::string &label) { +int64_t NumNodes(Client &client, const std::string &label) { auto result = ExecuteNTimesTillSuccess( client, "MATCH (n :" + label + ") RETURN COUNT(n) as cnt", {}, MAX_RETRIES); return result.first.records[0][0].ValueInt(); } -std::vector Neighbours(BoltClient &client, const std::string &label, +std::vector Neighbours(Client &client, const std::string &label, int64_t id) { auto result = ExecuteNTimesTillSuccess(client, "MATCH (n :" + label + @@ -244,8 +239,7 @@ std::vector Neighbours(BoltClient &client, const std::string &label, return ret; } -std::vector IndependentSet(BoltClient &client, - const std::string &label) { +std::vector IndependentSet(Client &client, const std::string &label) { const int64_t num_nodes = NumNodes(client, label); std::vector independent_nodes_ids; std::vector ids; @@ -282,8 +276,11 @@ int main(int argc, char **argv) { std::cin >> config; auto independent_nodes_ids = [&] { - BoltClient client(utils::ResolveHostname(FLAGS_address), FLAGS_port, - FLAGS_username, FLAGS_password); + Endpoint endpoint(utils::ResolveHostname(FLAGS_address), FLAGS_port); + Client client; + if (!client.Connect(endpoint, FLAGS_username, FLAGS_password)) { + LOG(FATAL) << "Couldn't connect to " << endpoint; + } return IndependentSet(client, INDEPENDENT_LABEL); }(); diff --git a/tests/macro_benchmark/clients/postgres_client.hpp b/tests/macro_benchmark/clients/postgres_client.hpp deleted file mode 100644 index 951125e81..000000000 --- a/tests/macro_benchmark/clients/postgres_client.hpp +++ /dev/null @@ -1,108 +0,0 @@ -#pragma once - -#include -#include -#include - -#include -#include -#include - -#include "communication/bolt/client.hpp" -#include "communication/bolt/v1/decoder/decoded_value.hpp" -#include "utils/exceptions.hpp" - -using communication::bolt::QueryData; - -namespace postgres { - -class ClientException : public utils::BasicException { - using utils::BasicException::BasicException; -}; - -class ClientQueryException : public ClientException { - public: - using ClientException::ClientException; - ClientQueryException() : ClientException("Couldn't execute query!") {} -}; - -class Client { - public: - Client(const std::string &host, const std::string &port, - const std::string &username, const std::string &password, - const std::string &database = "") { - // https://www.postgresql.org/docs/9.4/static/libpq-connect.html#LIBPQ-PARAMKEYWORDS - std::string pass = ""; - if (password != "") { - pass = "password=" + password; - } - std::string conninfo = - fmt::format("host={} port={} user={} {} dbname={} sslmode=disable", - host, port, username, pass, database); - - // Make a connection to the database. - connection_ = PQconnectdb(conninfo.c_str()); - - // Check to see that the backend connection was successfully made - if (PQstatus(connection_) != CONNECTION_OK) { - throw ClientException(PQerrorMessage(connection_)); - } - } - - QueryData Execute( - const std::string &query, - const std::map - ¶meters) { - QueryData ret; - - CHECK(parameters.size() == 0U) << "Parameters not yet supported"; - DLOG(INFO) << "Sending run message with statement: '" << query << "'"; - - result_ = PQexec(connection_, query.c_str()); - if (PQresultStatus(result_) == PGRES_TUPLES_OK) { - // get fields - int num_fields = PQnfields(result_); - for (int i = 0; i < num_fields; ++i) { - ret.fields.push_back(PQfname(result_, i)); - } - - // get records - int num_records = PQntuples(result_); - ret.records.resize(num_records); - for (int i = 0; i < num_records; ++i) { - for (int j = 0; j < num_fields; ++j) { - ret.records[i].push_back(std::string(PQgetvalue(result_, i, j))); - } - } - - // get metadata - ret.metadata.insert({"status", std::string(PQcmdStatus(result_))}); - ret.metadata.insert({"rows_affected", std::string(PQcmdTuples(result_))}); - } else if (PQresultStatus(result_) != PGRES_COMMAND_OK) { - throw ClientQueryException(PQerrorMessage(connection_)); - } - - PQclear(result_); - result_ = nullptr; - - return ret; - } - - void Close() { - if (result_ != nullptr) { - PQclear(result_); - result_ = nullptr; - } - if (connection_ != nullptr) { - PQfinish(connection_); - connection_ = nullptr; - } - } - - ~Client() { Close(); } - - private: - PGconn *connection_{nullptr}; - PGresult *result_{nullptr}; -}; -} diff --git a/tests/macro_benchmark/clients/query_client.cpp b/tests/macro_benchmark/clients/query_client.cpp index e4ef7d39f..fe82a57b9 100644 --- a/tests/macro_benchmark/clients/query_client.cpp +++ b/tests/macro_benchmark/clients/query_client.cpp @@ -4,27 +4,21 @@ #include #include -#include "communication/bolt/client.hpp" -#include "communication/bolt/v1/decoder/decoded_value.hpp" #include "threading/sync/spinlock.hpp" #include "utils/algorithm.hpp" #include "utils/string.hpp" #include "utils/timer.hpp" -#include "bolt_client.hpp" #include "common.hpp" -//#include "postgres_client.hpp" -DEFINE_string(protocol, "bolt", "Protocol to use (available: bolt, postgres)"); DEFINE_int32(num_workers, 1, "Number of workers"); DEFINE_string(input, "", "Input file"); DEFINE_string(output, "", "Output file"); DEFINE_string(address, "127.0.0.1", "Server address"); -DEFINE_int32(port, 0, "Server port"); +DEFINE_int32(port, 7687, "Server port"); DEFINE_string(username, "", "Username for the database"); DEFINE_string(password, "", "Password for the database"); -DEFINE_string(database, "", "Database for the database"); using communication::bolt::DecodedValue; @@ -49,11 +43,8 @@ void PrintSummary( os << "}\n"; } -template -void ExecuteQueries(const std::vector &queries, int num_workers, - std::ostream &ostream, std::string &address, uint16_t port, - std::string &username, std::string &password, - std::string &database) { +void ExecuteQueries(const std::vector &queries, + std::ostream &ostream) { std::vector threads; SpinLock spinlock; @@ -64,9 +55,13 @@ void ExecuteQueries(const std::vector &queries, int num_workers, utils::Timer timer; - for (int i = 0; i < num_workers; ++i) { + for (int i = 0; i < FLAGS_num_workers; ++i) { threads.push_back(std::thread([&]() { - ClientT client(address, port, username, password, database); + Endpoint endpoint(FLAGS_address, FLAGS_port); + Client client; + if (!client.Connect(endpoint, FLAGS_username, FLAGS_password)) { + LOG(FATAL) << "Couldn't connect to " << endpoint; + } std::string str; while (true) { @@ -91,7 +86,7 @@ void ExecuteQueries(const std::vector &queries, int num_workers, })); } - for (int i = 0; i < num_workers; ++i) { + for (int i = 0; i < FLAGS_num_workers; ++i) { threads[i].join(); } @@ -121,13 +116,6 @@ int main(int argc, char **argv) { ostream = &ofile; } - uint16_t port = FLAGS_port; - if (FLAGS_protocol == "bolt") { - if (port == 0) port = 7687; - } else if (FLAGS_protocol == "postgres") { - if (port == 0) port = 5432; - } - while (!istream->eof()) { std::vector queries; std::string query; @@ -135,27 +123,7 @@ int main(int argc, char **argv) { utils::Trim(query) != ";") { queries.push_back(query); } - - if (FLAGS_protocol == "bolt") { - ExecuteQueries(queries, FLAGS_num_workers, *ostream, - FLAGS_address, port, FLAGS_username, - FLAGS_password, FLAGS_database); - } else if (FLAGS_protocol == "postgres") { - LOG(FATAL) << "Postgres not yet supported"; - // TODO: Currently libpq is linked dynamically so it is a pain to move - // harness_client executable to other machines without libpq. - // CHECK(FLAGS_username != "") << "Username can't be empty for - // postgres!"; - // CHECK(FLAGS_database != "") << "Database can't be empty for - // postgres!"; - // if (port == "") port = "5432"; - // - // using PostgresClientT = postgres::Client; - // using PostgresExceptionT = postgres::ClientQueryException; - // ExecuteQueries( - // *istream, FLAGS_num_workers, *ostream, FLAGS_address, port, - // FLAGS_username, FLAGS_password, FLAGS_database); - } + ExecuteQueries(queries, *ostream); } return 0; diff --git a/tests/macro_benchmark/databases.py b/tests/macro_benchmark/databases.py index e6b6ccbcd..ba31a6911 100644 --- a/tests/macro_benchmark/databases.py +++ b/tests/macro_benchmark/databases.py @@ -128,48 +128,3 @@ class Neo: self.database_bin.wait() if os.path.exists(self.neo4j_home_path): shutil.rmtree(self.neo4j_home_path) - - -class Postgres: - """ - Knows how to start and stop PostgreSQL. - """ - def __init__(self, args, cpus): - self.log = logging.getLogger("PostgresRunner") - argp = ArgumentParser("PostgresArgumentParser") - argp.add_argument("--init-bin", default=get_absolute_path( - "postgresql/bin/initdb", "libs")) - argp.add_argument("--runner-bin", default=get_absolute_path( - "postgresql/bin/postgres", "libs")) - argp.add_argument("--port", default="5432", - help="Database and client port") - self.log.info("Initializing Runner with arguments %r", args) - self.args, _ = argp.parse_known_args(args) - self.username = "macro_benchmark" - self.database_bin = jail.get_process() - set_cpus("database-cpu-ids", self.database_bin, args) - - def start(self): - self.log.info("start") - self.data_path = tempfile.mkdtemp(dir="/dev/shm") - init_args = ["-D", self.data_path, "-U", self.username] - self.database_bin.run_and_wait(self.args.init_bin, init_args) - - # args - runner_args = ["-D", self.data_path, "-c", "port=" + self.args.port, - "-c", "ssl=false", "-c", "max_worker_processes=1"] - - try: - self.database_bin.run(self.args.runner_bin, args=runner_args, - timeout=600) - except: - shutil.rmtree(self.data_path) - raise Exception("Couldn't run PostgreSQL!") - - wait_for_server(self.args.port) - - def stop(self): - self.database_bin.send_signal(jail.SIGTERM) - self.database_bin.wait() - if os.path.exists(self.data_path): - shutil.rmtree(self.data_path)