tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add message with link utility (#243)

Browse Source
This commit is contained in:
Antonio Andelic 2021-10-07 05:51:30 -07:00 committed by GitHub
parent d7d291217b
commit 4e7ea34ae9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 156 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.githooks/pre-commit
View File

@ -30,6 +30,14 @@ for file in $modified_files; do
if [ $code -ne 0 ]; then
break
fi
echo "Running header checker..."
$project_folder/tools/header-checker.py $tmpdir/$file
code=$?
if [ $code -ne 0 ]; then
break
fi
done;
return $code

34
src/auth/auth.cpp
View File

@ -19,6 +19,7 @@
#include "utils/flag_validation.hpp"
#include "utils/license.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
#include "utils/settings.hpp"
#include "utils/string.hpp"
@ -102,17 +103,15 @@ std::optional<User> Auth::Authenticate(const std::string &username, const std::s
if (FLAGS_auth_module_create_missing_user) {
user = AddUser(username, password);
if (!user) {
spdlog::warn(
"Couldn't authenticate user '{}' using the auth module because "
"the user already exists as a role!",
username);
spdlog::warn(utils::MessageWithLink(
"Couldn't create the missing user '{}' using the auth module because the user already exists as a role.",
username, "https://memgr.ph/auth"));
return std::nullopt;
}
} else {
spdlog::warn(
"Couldn't authenticate user '{}' using the auth module because the "
"user doesn't exist!",
username);
spdlog::warn(utils::MessageWithLink(
"Couldn't authenticate user '{}' using the auth module because the user doesn't exist.", username,
"https://memgr.ph/auth"));
return std::nullopt;
}
} else {
@ -126,17 +125,16 @@ std::optional<User> Auth::Authenticate(const std::string &username, const std::s
role = AddRole(rolename);
if (!role) {
spdlog::warn(
"Couldn't authenticate user '{}' using the auth module "
"because the user's role '{}' already exists as a user!",
username, rolename);
utils::MessageWithLink("Couldn't authenticate user '{}' using the auth module because the user's "
"role '{}' already exists as a user.",
username, rolename, "https://memgr.ph/auth"));
return std::nullopt;
}
SaveRole(*role);
} else {
spdlog::warn(
"Couldn't authenticate user '{}' using the auth module because "
"the user's role '{}' doesn't exist!",
username, rolename);
spdlog::warn(utils::MessageWithLink(
"Couldn't authenticate user '{}' using the auth module because the user's role '{}' doesn't exist.",
username, rolename, "https://memgr.ph/auth"));
return std::nullopt;
}
}
@ -150,11 +148,13 @@ std::optional<User> Auth::Authenticate(const std::string &username, const std::s
} else {
auto user = GetUser(username);
if (!user) {
spdlog::warn("Couldn't authenticate user '{}' because the user doesn't exist", username);
spdlog::warn(utils::MessageWithLink("Couldn't authenticate user '{}' because the user doesn't exist.", username,
"https://memgr.ph/auth"));
return std::nullopt;
}
if (!user->CheckPassword(password)) {
spdlog::warn("Couldn't authenticate user '{}'", username);
spdlog::warn(utils::MessageWithLink("Couldn't authenticate user '{}' because the password is not correct.",
username, "https://memgr.ph/auth"));
return std::nullopt;
}
return user;

4
src/communication/bolt/v1/states/executing.hpp
View File

@ -23,6 +23,7 @@
#include "communication/exceptions.hpp"
#include "utils/likely.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
namespace communication::bolt {
@ -64,7 +65,8 @@ inline std::pair<std::string, std::string> ExceptionToErrorMessage(const std::ex
// All exceptions used in memgraph are derived from BasicException. Since
// we caught some other exception we don't know what is going on. Return
// DatabaseError, log real message and return generic string.
spdlog::error("Unknown exception occurred during query execution {}", e.what());
spdlog::error(
utils::MessageWithLink("Unknown exception occurred during query execution {}.", e.what(), "https://memgr.ph/unknown"));
return {"Memgraph.DatabaseError.MemgraphError.MemgraphError",
"An unknown exception occurred, this is unexpected. Real message "
"should be in database logs."};

5
src/communication/server.hpp
View File

@ -24,6 +24,7 @@
#include "communication/listener.hpp"
#include "io/network/socket.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
#include "utils/thread.hpp"
namespace communication {
@ -88,13 +89,13 @@ class Server final {
alive_.store(true);
if (!socket_.Bind(endpoint_)) {
spdlog::error("Cannot bind to socket on {}", endpoint_);
spdlog::error(utils::MessageWithLink("Cannot bind to socket on endpoint {}.", endpoint_, "https://memgr.ph/socket"));
alive_.store(false);
return false;
}
socket_.SetTimeout(1, 0);
if (!socket_.Listen(1024)) {
spdlog::error("Cannot listen on socket {}", endpoint_);
spdlog::error(utils::MessageWithLink("Cannot listen on socket {}", endpoint_, "https://memgr.ph/socket"));
alive_.store(false);
return false;
}

9
src/communication/session.hpp
View File

@ -29,6 +29,7 @@
#include "io/network/socket.hpp"
#include "io/network/stream_buffer.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
#include "utils/on_scope_exit.hpp"
#include "utils/spin_lock.hpp"
@ -190,10 +191,10 @@ class Session final {
throw utils::BasicException(SslGetLastError());
} else {
// This is a fatal error.
spdlog::error(
"An unknown error occured while processing SSL message."
" Please make sure that you have SSL properly configured on "
"the server and the client.");
spdlog::error(utils::MessageWithLink(
"An unknown error occurred while processing SSL messages. "
"Please make sure that you have SSL properly configured on the server and the client.",
"https://memgr.ph/ssl"));
throw utils::BasicException(SslGetLastError());
}
} else if (len == 0) {

9
src/io/network/endpoint.cpp
View File

@ -18,6 +18,7 @@
#include "io/network/endpoint.hpp"
#include "io/network/network_error.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
#include "utils/string.hpp"
namespace io::network {
@ -65,15 +66,17 @@ std::optional<std::pair<std::string, uint16_t>> Endpoint::ParseSocketOrIpAddress
try {
int_port = utils::ParseInt(parts[1]);
} catch (utils::BasicException &e) {
spdlog::error("Invalid port number: {}", parts[1]);
spdlog::error(utils::MessageWithLink("Invalid port number {}.", parts[1], "https://memgr.ph/ports"));
return std::nullopt;
}
if (int_port < 0) {
spdlog::error("Port number must be a positive integer!");
spdlog::error(utils::MessageWithLink("Invalid port number {}. The port number must be a positive integer.",
int_port, "https://memgr.ph/ports"));
return std::nullopt;
}
if (int_port > std::numeric_limits<uint16_t>::max()) {
spdlog::error("Port number exceeded maximum possible size!");
spdlog::error(utils::MessageWithLink("Invalid port number. The port number exceedes the maximum possible size.",
"https://memgr.ph/ports"));
return std::nullopt;
}

15
src/memgraph.cpp
View File

@ -53,6 +53,7 @@
#include "utils/license.hpp"
#include "utils/logging.hpp"
#include "utils/memory_tracker.hpp"
#include "utils/message.hpp"
#include "utils/readable_size.hpp"
#include "utils/rw_lock.hpp"
#include "utils/settings.hpp"
@ -998,13 +999,16 @@ int main(int argc, char **argv) {
auto gil = py::EnsureGIL();
auto maybe_exc = py::AppendToSysPath(py_support_dir.c_str());
if (maybe_exc) {
spdlog::error("Unable to load support for embedded Python: {}", *maybe_exc);
spdlog::error(
utils::MessageWithLink("Unable to load support for embedded Python: {}.", *maybe_exc, "https://memgr.ph/python"));
}
} else {
spdlog::error("Unable to load support for embedded Python: missing directory {}", py_support_dir);
spdlog::error(utils::MessageWithLink("Unable to load support for embedded Python: missing directory {}.",
py_support_dir, "https://memgr.ph/python"));
}
} catch (const std::filesystem::filesystem_error &e) {
spdlog::error("Unable to load support for embedded Python: {}", e.what());
spdlog::error(
utils::MessageWithLink("Unable to load support for embedded Python: {}.", e.what(), "https://memgr.ph/python"));
}
// Initialize the communication library.
@ -1021,7 +1025,8 @@ int main(int argc, char **argv) {
mem_log_scheduler.Run("Memory warning", std::chrono::seconds(3), [] {
auto free_ram = utils::sysinfo::AvailableMemory();
if (free_ram && *free_ram / 1024 < FLAGS_memory_warning_threshold)
spdlog::warn("Running out of available RAM, only {} MB left", *free_ram / 1024);
spdlog::warn(utils::MessageWithLink("Running out of available RAM, only {} MB left.", *free_ram / 1024,
"https://memgr.ph/ram"));
});
} else {
// Kernel version for the `MemAvailable` value is from: man procfs
@ -1152,7 +1157,7 @@ int main(int argc, char **argv) {
service_name = "BoltS";
spdlog::info("Using secure Bolt connection (with SSL)");
} else {
spdlog::warn("Using non-secure Bolt connection (without SSL)");
spdlog::warn(utils::MessageWithLink("Using non-secure Bolt connection (without SSL).", "https://memgr.ph/ssl"));
}
ServerT server({FLAGS_bolt_address, static_cast<uint16_t>(FLAGS_bolt_port)}, &session_data, &context,

8
src/mg_import_csv.cpp
View File

@ -23,6 +23,7 @@
#include "storage/v2/storage.hpp"
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
#include "utils/string.hpp"
#include "utils/timer.hpp"
#include "version.hpp"
@ -433,7 +434,7 @@ void ProcessNodeRow(storage::Storage *store, const std::vector<Field> &fields, c
auto it = node_id_map->find(node_id);
if (it != node_id_map->end()) {
if (FLAGS_skip_duplicate_nodes) {
spdlog::warn("Skipping duplicate node with ID '{}'", node_id);
spdlog::warn(utils::MessageWithLink("Skipping duplicate node with ID '{}'.", node_id, "https://memgr.ph/csv"));
return;
} else {
throw LoadException("Node with ID '{}' already exists", node_id);
@ -524,7 +525,8 @@ void ProcessRelationshipsRow(storage::Storage *store, const std::vector<Field> &
auto it = node_id_map.find(node_id);
if (it == node_id_map.end()) {
if (FLAGS_skip_bad_relationships) {
spdlog::warn("Skipping bad relationship with START_ID '{}'", node_id);
spdlog::warn(
utils::MessageWithLink("Skipping bad relationship with START_ID '{}'.", node_id, "https://memgr.ph/csv"));
return;
} else {
throw LoadException("Node with ID '{}' does not exist", node_id);
@ -541,7 +543,7 @@ void ProcessRelationshipsRow(storage::Storage *store, const std::vector<Field> &
auto it = node_id_map.find(node_id);
if (it == node_id_map.end()) {
if (FLAGS_skip_bad_relationships) {
spdlog::warn("Skipping bad relationship with END_ID '{}'", node_id);
spdlog::warn(utils::MessageWithLink("Skipping bad relationship with END_ID '{}'.", node_id, "https://memgr.ph/csv"));
return;
} else {
throw LoadException("Node with ID '{}' does not exist", node_id);

21
src/query/procedure/module.cpp
View File

@ -24,6 +24,7 @@ extern "C" {
#include "query/procedure/py_module.hpp"
#include "utils/file.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
#include "utils/pmr/vector.hpp"
#include "utils/string.hpp"
@ -324,14 +325,15 @@ bool SharedLibraryModule::Load(const std::filesystem::path &file_path) {
dlerror(); // Clear any existing error.
handle_ = dlopen(file_path.c_str(), RTLD_NOW | RTLD_LOCAL);
if (!handle_) {
spdlog::error("Unable to load module {}; {}", file_path, dlerror());
// NOLINTNEXTLINE(concurrency-mt-unsafe)
spdlog::error(utils::MessageWithLink("Unable to load module {}; {}.", file_path, dlerror(), "https://memgr.ph/modules"));
return false;
}
// Get required mgp_init_module
init_fn_ = reinterpret_cast<int (*)(mgp_module *, mgp_memory *)>(dlsym(handle_, "mgp_init_module"));
char *dl_errored = dlerror();
if (!init_fn_ || dl_errored) {
spdlog::error("Unable to load module {}; {}", file_path, dl_errored);
spdlog::error(utils::MessageWithLink("Unable to load module {}; {}.", file_path, dl_errored, "https://memgr.ph/modules"));
dlclose(handle_);
handle_ = nullptr;
return false;
@ -382,7 +384,8 @@ bool SharedLibraryModule::Close() {
spdlog::warn("When closing module {}; mgp_shutdown_module returned {}", file_path_, shutdown_res);
}
if (dlclose(handle_) != 0) {
spdlog::error("Failed to close module {}; {}", file_path_, dlerror());
// NOLINTNEXTLINE(concurrency-mt-unsafe)
spdlog::error(utils::MessageWithLink("Failed to close module {}; {}.", file_path_, dlerror(), "https://memgr.ph/modules"));
return false;
}
spdlog::info("Closed module {}", file_path_);
@ -441,7 +444,7 @@ bool PythonModule::Load(const std::filesystem::path &file_path) {
auto gil = py::EnsureGIL();
auto maybe_exc = py::AppendToSysPath(file_path.parent_path().c_str());
if (maybe_exc) {
spdlog::error("Unable to load module {}; {}", file_path, *maybe_exc);
spdlog::error(utils::MessageWithLink("Unable to load module {}; {}.", file_path, *maybe_exc, "https://memgr.ph/modules"));
return false;
}
bool succ = true;
@ -466,7 +469,7 @@ bool PythonModule::Load(const std::filesystem::path &file_path) {
return true;
}
auto exc_info = py::FetchError().value();
spdlog::error("Unable to load module {}; {}", file_path, exc_info);
spdlog::error(utils::MessageWithLink("Unable to load module {}; {}.", file_path, exc_info, "https://memgr.ph/modules"));
return false;
}
@ -509,7 +512,7 @@ namespace {
std::unique_ptr<Module> LoadModuleFromFile(const std::filesystem::path &path) {
const auto &ext = path.extension();
if (ext != ".so" && ext != ".py") {
spdlog::warn("Unknown query module file {}", path);
spdlog::warn(utils::MessageWithLink("Unknown query module file {}.", path, "https://memgr.ph/modules"));
return nullptr;
}
std::unique_ptr<Module> module;
@ -531,7 +534,7 @@ bool ModuleRegistry::RegisterModule(const std::string_view &name, std::unique_pt
MG_ASSERT(!name.empty(), "Module name cannot be empty");
MG_ASSERT(module, "Tried to register an invalid module");
if (modules_.find(name) != modules_.end()) {
spdlog::error("Unable to overwrite an already loaded module {}", name);
spdlog::error(utils::MessageWithLink("Unable to overwrite an already loaded module {}.", name, "https://memgr.ph/modules"));
return false;
}
modules_.emplace(name, std::move(module));
@ -561,7 +564,7 @@ void ModuleRegistry::SetModulesDirectory(std::vector<std::filesystem::path> modu
bool ModuleRegistry::LoadModuleIfFound(const std::filesystem::path &modules_dir, const std::string_view name) {
if (!utils::DirExists(modules_dir)) {
spdlog::error("Module directory {} doesn't exist", modules_dir);
spdlog::error(utils::MessageWithLink("Module directory {} doesn't exist.", modules_dir, "https://memgr.ph/modules"));
return false;
}
for (const auto &entry : std::filesystem::directory_iterator(modules_dir)) {
@ -598,7 +601,7 @@ bool ModuleRegistry::LoadOrReloadModuleFromName(const std::string_view name) {
void ModuleRegistry::LoadModulesFromDirectory(const std::filesystem::path &modules_dir) {
if (modules_dir.empty()) return;
if (!utils::DirExists(modules_dir)) {
spdlog::error("Module directory {} doesn't exist", modules_dir);
spdlog::error(utils::MessageWithLink("Module directory {} doesn't exist.", modules_dir, "https://memgr.ph/modules"));
return;
}
for (const auto &entry : std::filesystem::directory_iterator(modules_dir)) {

8
src/storage/v2/durability/durability.cpp
View File

@ -29,6 +29,7 @@
#include "storage/v2/durability/wal.hpp"
#include "utils/logging.hpp"
#include "utils/memory_tracker.hpp"
#include "utils/message.hpp"
namespace storage::durability {
@ -168,7 +169,8 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
spdlog::info("Recovering persisted data using snapshot ({}) and WAL directory ({}).", snapshot_directory,
wal_directory);
if (!utils::DirExists(snapshot_directory) && !utils::DirExists(wal_directory)) {
spdlog::warn("Snapshot or WAL directory don't exist, there is nothing to recover.");
spdlog::warn(utils::MessageWithLink("Snapshot or WAL directory don't exist, there is nothing to recover.",
"https://memgr.ph/durability"));
return std::nullopt;
}
@ -242,7 +244,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
}
MG_ASSERT(!error_code, "Couldn't recover data because an error occurred: {}!", error_code.message());
if (wal_files.empty()) {
spdlog::warn("No snapshot or WAL file found!");
spdlog::warn(utils::MessageWithLink("No snapshot or WAL file found.", "https://memgr.ph/durability"));
return std::nullopt;
}
std::sort(wal_files.begin(), wal_files.end());
@ -254,7 +256,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
auto maybe_wal_files = GetWalFiles(wal_directory, *uuid);
if (!maybe_wal_files) {
spdlog::warn("Couldn't get WAL file info from the WAL directory!");
spdlog::warn(utils::MessageWithLink("Couldn't get WAL file info from the WAL directory.", "https://memgr.ph/durability"));
return std::nullopt;
}

12
src/storage/v2/durability/snapshot.cpp
View File

@ -22,6 +22,7 @@
#include "storage/v2/vertex_accessor.hpp"
#include "utils/file_locker.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
namespace storage::durability {
@ -913,9 +914,8 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
if (error_code) {
spdlog::error(
"Couldn't ensure that exactly {} snapshots exist because an error "
"occurred: {}",
snapshot_retention_count, error_code.message());
utils::MessageWithLink("Couldn't ensure that exactly {} snapshots exist because an error occurred: {}.",
snapshot_retention_count, error_code.message(), "https://memgr.ph/snapshots"));
}
std::sort(old_snapshot_files.begin(), old_snapshot_files.end());
if (old_snapshot_files.size() > snapshot_retention_count - 1) {
@ -945,9 +945,9 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
if (error_code) {
spdlog::error(
"Couldn't ensure that only the absolutely necessary WAL files exist "
"because an error occurred: {}",
error_code.message());
utils::MessageWithLink("Couldn't ensure that only the absolutely necessary WAL files exist "
"because an error occurred: {}.",
error_code.message(), "https://memgr.ph/snapshots"));
}
std::sort(wal_files.begin(), wal_files.end());
uint64_t snapshot_start_timestamp = transaction->start_timestamp;

6
src/storage/v2/replication/replication_client.cpp
View File

@ -20,6 +20,7 @@
#include "storage/v2/transaction.hpp"
#include "utils/file_locker.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
namespace storage {
@ -105,12 +106,13 @@ void Storage::ReplicationClient::TryInitializeClient() {
} catch (const rpc::RpcFailedException &) {
std::unique_lock client_guarde{client_lock_};
replica_state_.store(replication::ReplicaState::INVALID);
spdlog::error("Failed to connect to replica {} at {}", name_, rpc_client_->Endpoint());
spdlog::error(utils::MessageWithLink("Failed to connect to replica {} at the endpoint {}.", name_,
rpc_client_->Endpoint(), "https://memgr.ph/replication"));
}
}
void Storage::ReplicationClient::HandleRpcFailure() {
spdlog::error("Couldn't replicate data to {}", name_);
spdlog::error(utils::MessageWithLink("Couldn't replicate data to {}.", name_, "https://memgr.ph/replication"));
thread_pool_.AddTask([this] {
rpc_client_->Abort();
this->TryInitializeClient();

5
src/storage/v2/storage.cpp
View File

@ -33,6 +33,7 @@
#include "utils/file.hpp"
#include "utils/logging.hpp"
#include "utils/memory_tracker.hpp"
#include "utils/message.hpp"
#include "utils/rw_lock.hpp"
#include "utils/spin_lock.hpp"
#include "utils/stat.hpp"
@ -376,7 +377,7 @@ Storage::Storage(Config config)
if (auto maybe_error = this->CreateSnapshot(); maybe_error.HasError()) {
switch (maybe_error.GetError()) {
case CreateSnapshotError::DisabledForReplica:
spdlog::warn("Snapshots are disabled for replicas!");
spdlog::warn(utils::MessageWithLink("Snapshots are disabled for replicas.", "https://memgr.ph/replication"));
break;
}
}
@ -413,7 +414,7 @@ Storage::~Storage() {
if (auto maybe_error = this->CreateSnapshot(); maybe_error.HasError()) {
switch (maybe_error.GetError()) {
case CreateSnapshotError::DisabledForReplica:
spdlog::warn("Snapshots are disabled for replicas!");
spdlog::warn(utils::MessageWithLink("Snapshots are disabled for replicas.", "https://memgr.ph/replication"));
break;
}
}

23
src/utils/message.hpp Normal file
View File

@ -0,0 +1,23 @@
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <fmt/format.h>
namespace utils {
template <typename... Args>
std::string MessageWithLink(const std::string_view format, Args &&...args) {
return fmt::format(fmt::format("{} For more details, visit {{}}.", format), std::forward<Args>(args)...);
}
} // namespace utils

46
tools/header-checker.py Executable file
View File

@ -0,0 +1,46 @@
#!/usr/bin/python3
import argparse
import sys
BSL_HEADER = """// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt."""
MEL_HEADER = """// Copyright 2021 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
// this file except in compliance with the License. You may obtain a copy of the License at https://memgraph.com/legal.
"""
def main():
parser = argparse.ArgumentParser()
parser.add_argument("file", help="directory with source files", nargs="?")
args = parser.parse_args()
with open(args.file, 'r') as f:
content = f.read()
has_header = content.startswith(BSL_HEADER) or content.startswith(MEL_HEADER)
if not has_header:
def red(s):
return f"\x1b[31m{s}\x1b[0m"
sys.stdout.writelines(red("The file is missing a header. Please add the BSL or MEL license header!\n"))
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main()