Improve Jepsen run script (#64)
Co-authored-by: Antonio Andelic <antonio.andelic@memgraph.io>
This commit is contained in:
Marko Budiselić
Antonio Andelic
parent
b7bbd026de
commit
9966ba1d52
9
.github/workflows/diff.yaml
vendored
9
.github/workflows/diff.yaml
vendored
@ -352,16 +352,11 @@ jobs:
|
||||
- name: Run Jepsen tests
|
||||
run: |
|
||||
cd tests/jepsen
|
||||
./setup-local-docker-cluster.sh copy ../../build/memgraph
|
||||
# NOTE: docker exec -t is NOT ok because gh user does NOT have TTY.
|
||||
# NOTE: ~/.bashrc has to be manually sourced when bash -c is used
|
||||
# because some Jepsen config is there.
|
||||
docker exec jepsen-control bash -c "source ~/.bashrc && cd memgraph && lein run test-all --local-binary /opt/memgraph/memgraph --node-configs resources/node-config.edn"
|
||||
docker exec jepsen-control bash -c 'tar -czvf /jepsen/memgraph/Jepsen.tar.gz $(readlink -f /jepsen/memgraph/store/latest)'
|
||||
docker cp jepsen-control:/jepsen/memgraph/Jepsen.tar.gz ./
|
||||
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
|
||||
|
||||
- name: Save Jepsen report
|
||||
uses: actions/upload-artifact@v2
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: "Jepsen Report"
|
||||
path: tests/jepsen/Jepsen.tar.gz
|
||||
|
||||
@ -6,8 +6,8 @@
|
||||
|
||||
#include "glog/logging.h"
|
||||
|
||||
#include "utils/string.hpp"
|
||||
#include "io/network/endpoint.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace io::network {
|
||||
|
||||
|
||||
@ -5,11 +5,11 @@
|
||||
#include <glog/logging.h>
|
||||
|
||||
#include "glue/communication.hpp"
|
||||
#include "query/constants.hpp"
|
||||
#include "query/context.hpp"
|
||||
#include "query/db_accessor.hpp"
|
||||
#include "query/dump.hpp"
|
||||
#include "query/exceptions.hpp"
|
||||
#include "query/constants.hpp"
|
||||
#include "query/frontend/ast/cypher_main_visitor.hpp"
|
||||
#include "query/frontend/opencypher/parser.hpp"
|
||||
#include "query/frontend/semantic/required_privileges.hpp"
|
||||
@ -222,20 +222,20 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
|
||||
}
|
||||
}
|
||||
|
||||
auto maybe_ip_and_port = io::network::Endpoint::ParseSocketOrIpAddress(
|
||||
socket_address, query::kDefaultReplicationPort);
|
||||
if (maybe_ip_and_port) {
|
||||
auto [ip, port] = *maybe_ip_and_port;
|
||||
auto ret =
|
||||
db_->RegisterReplica(name, {std::move(ip), port}, repl_mode,
|
||||
{.timeout = timeout, .ssl = std::nullopt});
|
||||
if (ret.HasError()) {
|
||||
throw QueryRuntimeException(
|
||||
fmt::format("Couldn't register replica '{}'!", name));
|
||||
}
|
||||
} else {
|
||||
throw QueryRuntimeException("Invalid socket address!");
|
||||
auto maybe_ip_and_port = io::network::Endpoint::ParseSocketOrIpAddress(
|
||||
socket_address, query::kDefaultReplicationPort);
|
||||
if (maybe_ip_and_port) {
|
||||
auto [ip, port] = *maybe_ip_and_port;
|
||||
auto ret =
|
||||
db_->RegisterReplica(name, {std::move(ip), port}, repl_mode,
|
||||
{.timeout = timeout, .ssl = std::nullopt});
|
||||
if (ret.HasError()) {
|
||||
throw QueryRuntimeException(
|
||||
fmt::format("Couldn't register replica '{}'!", name));
|
||||
}
|
||||
} else {
|
||||
throw QueryRuntimeException("Invalid socket address!");
|
||||
}
|
||||
}
|
||||
|
||||
/// @throw QueryRuntimeException if an error ocurred.
|
||||
|
||||
179
tests/jepsen/run.sh
Executable file
179
tests/jepsen/run.sh
Executable file
@ -0,0 +1,179 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
|
||||
HELP_EXIT() {
|
||||
echo ""
|
||||
echo "HELP: $0 help|cluster-up|test [args]"
|
||||
echo ""
|
||||
echo " test args --binary MEMGRAPH_BINARY_PATH"
|
||||
echo " --ignore-run-stdout-logs Ignore lein run stdout logs."
|
||||
echo " --ignore-run-stderr-logs Ignore lein run stderr logs."
|
||||
echo " --nodes-no JEPSEN_ACTIVE_NODES_NO"
|
||||
echo " --run-args \"CONTROL_LEIN_RUN_ARGS\" (NOTE: quotes)"
|
||||
echo ""
|
||||
exit 1
|
||||
}
|
||||
|
||||
ERROR() {
|
||||
/bin/echo -e "\e[101m\e[97m[ERROR]\e[49m\e[39m" "$@"
|
||||
}
|
||||
|
||||
INFO() {
|
||||
/bin/echo -e "\e[104m\e[97m[INFO]\e[49m\e[39m" "$@"
|
||||
}
|
||||
|
||||
if ! command -v docker > /dev/null 2>&1 || ! command -v docker-compose > /dev/null 2>&1; then
|
||||
ERROR "docker and docker-compose have to be installed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
MEMGRAPH_BINARY_PATH="../../build/memgraph"
|
||||
JEPSEN_ACTIVE_NODES_NO=5
|
||||
CONTROL_LEIN_RUN_ARGS="test-all --node-configs resources/node-config.edn"
|
||||
CONTROL_LEIN_RUN_STDOUT_LOGS=1
|
||||
CONTROL_LEIN_RUN_STDERR_LOGS=1
|
||||
|
||||
if [ ! -d "$script_dir/jepsen" ]; then
|
||||
git clone https://github.com/jepsen-io/jepsen.git -b "0.2.1" "$script_dir/jepsen"
|
||||
fi
|
||||
|
||||
if [ "$#" -lt 1 ]; then
|
||||
HELP_EXIT
|
||||
fi
|
||||
|
||||
# Initialize testing context by copying source/binary files. Inside CI,
|
||||
# Memgraph is tested on a single machine cluster based on Docker containers.
|
||||
# Once these tests will be part of the official Jepsen repo, the majority of
|
||||
# functionalities inside this script won't be needed because each node clones
|
||||
# the public repo.
|
||||
case $1 in
|
||||
help)
|
||||
HELP_EXIT
|
||||
;;
|
||||
# Start Jepsen Docker cluster of 5 nodes. To configure the cluster please
|
||||
# take a look under jepsen/docker/docker-compose.yml.
|
||||
# NOTE: If you delete the jepsen folder where docker config is located,
|
||||
# the current cluster is broken because it relies on the folder. That can
|
||||
# happen easiliy because the jepsen folder is git ignored.
|
||||
cluster-up)
|
||||
"$script_dir/jepsen/docker/bin/up" --daemon
|
||||
;;
|
||||
# Run tests against the specified Memgraph binary.
|
||||
test)
|
||||
shift
|
||||
while [[ $# -gt 0 ]]; do
|
||||
key="$1"
|
||||
case $key in
|
||||
--binary)
|
||||
shift
|
||||
MEMGRAPH_BINARY_PATH="$1"
|
||||
shift
|
||||
;;
|
||||
--ignore-run-stdout-logs)
|
||||
CONTROL_LEIN_RUN_STDOUT_LOGS=0
|
||||
shift
|
||||
;;
|
||||
--ignore-run-stderr-logs)
|
||||
CONTROL_LEIN_RUN_STDERR_LOGS=0
|
||||
shift
|
||||
;;
|
||||
--nodes-no)
|
||||
shift
|
||||
JEPSEN_ACTIVE_NODES_NO="$1"
|
||||
shift
|
||||
;;
|
||||
--run-args)
|
||||
shift
|
||||
CONTROL_LEIN_RUN_ARGS="$1"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
ERROR "Unknown option $1."
|
||||
HELP_EXIT
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Resolve binary path if it is a link.
|
||||
binary_path="$MEMGRAPH_BINARY_PATH"
|
||||
if [ -L "$binary_path" ]; then
|
||||
binary_path=$(readlink "$binary_path")
|
||||
fi
|
||||
binary_name=$(basename -- "$binary_path")
|
||||
|
||||
# Copy Memgraph binary.
|
||||
for iter in $(seq 1 "$JEPSEN_ACTIVE_NODES_NO"); do
|
||||
jepsen_node_name="jepsen-n$iter"
|
||||
docker exec "$jepsen_node_name" mkdir -p /opt/memgraph
|
||||
docker cp "$binary_path" "$jepsen_node_name":/opt/memgraph/"$binary_name"
|
||||
docker exec "$jepsen_node_name" bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph"
|
||||
INFO "Copying $binary_name to $jepsen_node_name DONE."
|
||||
done
|
||||
|
||||
# Copy test files into the control node.
|
||||
docker exec jepsen-control mkdir -p /jepsen/memgraph
|
||||
docker cp "$script_dir/src/." jepsen-control:/jepsen/memgraph/src/
|
||||
docker cp "$script_dir/test/." jepsen-control:/jepsen/memgraph/test/
|
||||
docker cp "$script_dir/resources/." jepsen-control:/jepsen/memgraph/resources/
|
||||
docker cp "$script_dir/project.clj" jepsen-control:/jepsen/memgraph/project.clj
|
||||
INFO "Copying test files to jepsen-control DONE."
|
||||
|
||||
start_time="$(docker exec jepsen-control bash -c 'date -u +"%Y%m%dT%H%M%S"').000Z"
|
||||
|
||||
# Run the test.
|
||||
# NOTE: docker exec -t is NOT ok because gh CI user does NOT have TTY.
|
||||
# NOTE: ~/.bashrc has to be manually sourced when bash -c is used
|
||||
# because some Jepsen config is there.
|
||||
set +e
|
||||
if [ "$CONTROL_LEIN_RUN_STDOUT_LOGS" -eq 0 ]; then
|
||||
redirect_stdout_logs="/dev/null"
|
||||
else
|
||||
redirect_stdout_logs="/dev/stdout"
|
||||
fi
|
||||
if [ "$CONTROL_LEIN_RUN_STDERR_LOGS" -eq 0 ]; then
|
||||
redirect_stderr_logs="/dev/null"
|
||||
else
|
||||
redirect_stderr_logs="/dev/stderr"
|
||||
fi
|
||||
INFO "Jepsen run in progress... START_TIME: $start_time"
|
||||
docker exec jepsen-control bash -c "source ~/.bashrc && cd memgraph && lein run $CONTROL_LEIN_RUN_ARGS" 1> $redirect_stdout_logs 2> $redirect_stderr_logs
|
||||
# To be able to archive the run result even if the run fails.
|
||||
jepsen_run_exit_status=$?
|
||||
end_time="$(docker exec jepsen-control bash -c 'date -u +"%Y%m%dT%H%M%S"').000Z"
|
||||
INFO "Jepsen run DONE. END_TIME: $end_time"
|
||||
set -e
|
||||
|
||||
# Pack all test workload runs between start and end time.
|
||||
all_workloads=$(docker exec jepsen-control bash -c 'ls /jepsen/memgraph/store/' | grep test-)
|
||||
all_workload_run_folders=""
|
||||
for workload in $all_workloads; do
|
||||
for time_folder in $(docker exec jepsen-control bash -c "ls /jepsen/memgraph/store/$workload"); do
|
||||
if [[ "$time_folder" == "latest" ]]; then
|
||||
continue
|
||||
fi
|
||||
# The early continue pattern here is nice because bash doesn't
|
||||
# have >= for the string comparison (marginal values).
|
||||
if [[ "$time_folder" < "$start_time" ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ "$time_folder" > "$end_time" ]]; then
|
||||
continue
|
||||
fi
|
||||
all_workload_run_folders="$all_workload_run_folders /jepsen/memgraph/store/$workload/$time_folder"
|
||||
done
|
||||
done
|
||||
docker exec jepsen-control bash -c "tar -czvf /jepsen/memgraph/Jepsen.tar.gz $all_workload_run_folders"
|
||||
docker cp jepsen-control:/jepsen/memgraph/Jepsen.tar.gz ./
|
||||
INFO "Test and results packing DONE."
|
||||
|
||||
# If the run has failed, this script also has to return non-zero status.
|
||||
if [ "$jepsen_run_exit_status" -ne 0 ]; then
|
||||
exit "$jepsen_run_exit_status"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
HELP_EXIT
|
||||
;;
|
||||
esac
|
||||
@ -1,55 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
|
||||
# NOTE: docker and docker-compose have to be installed on the system.
|
||||
|
||||
if [ ! -d "$script_dir/jepsen" ]; then
|
||||
git clone https://github.com/jepsen-io/jepsen.git -b "0.2.1" "$script_dir/jepsen"
|
||||
fi
|
||||
|
||||
# Initialize testing context by copying source/binary files. Inside CI,
|
||||
# Memgraph is tested on a single machine cluster based on Docker containers.
|
||||
# Once these tests will be part of the official Jepsen repo, the majority of
|
||||
# functionalities inside this script won't be needed because each node clones
|
||||
# the public repo.
|
||||
case $1 in
|
||||
# Start Jepsen Docker cluster.
|
||||
up)
|
||||
"$script_dir/jepsen/docker/bin/up" --daemon
|
||||
;;
|
||||
# Copy Memgraph Jepsen project files and Memgraph binary if specified.
|
||||
copy)
|
||||
shift 1
|
||||
binary_path=$1
|
||||
if [ -L "$binary_path" ]; then
|
||||
binary_path=$(readlink "$binary_path")
|
||||
fi
|
||||
binary_name=$(basename -- "$binary_path")
|
||||
|
||||
# Copy Memgraph build binary.
|
||||
docker exec jepsen-n1 mkdir -p /opt/memgraph
|
||||
docker exec jepsen-n2 mkdir -p /opt/memgraph
|
||||
docker exec jepsen-n3 mkdir -p /opt/memgraph
|
||||
docker exec jepsen-n4 mkdir -p /opt/memgraph
|
||||
docker exec jepsen-n5 mkdir -p /opt/memgraph
|
||||
docker cp "$binary_path" jepsen-n1:/opt/memgraph/"$binary_name"
|
||||
docker exec jepsen-n1 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph"
|
||||
docker cp "$binary_path" jepsen-n2:/opt/memgraph/"$binary_name"
|
||||
docker exec jepsen-n2 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph"
|
||||
docker cp "$binary_path" jepsen-n3:/opt/memgraph/"$binary_name"
|
||||
docker exec jepsen-n3 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph"
|
||||
docker cp "$binary_path" jepsen-n4:/opt/memgraph/"$binary_name"
|
||||
docker exec jepsen-n4 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph"
|
||||
docker cp "$binary_path" jepsen-n5:/opt/memgraph/"$binary_name"
|
||||
docker exec jepsen-n5 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph"
|
||||
|
||||
# Copy tests/jepsen/memgraph required files into the control node.
|
||||
docker exec jepsen-control mkdir -p /jepsen/memgraph
|
||||
docker cp "$script_dir/src/." jepsen-control:/jepsen/memgraph/src/
|
||||
docker cp "$script_dir/test/." jepsen-control:/jepsen/memgraph/test/
|
||||
docker cp "$script_dir/resources/." jepsen-control:/jepsen/memgraph/resources/
|
||||
docker cp "$script_dir/project.clj" jepsen-control:/jepsen/memgraph/project.clj
|
||||
;;
|
||||
esac
|
||||
@ -164,7 +164,7 @@
|
||||
workloads (if-let [w (:workload opts)] [w] (keys workloads))
|
||||
node-configs (if (:node-configs opts)
|
||||
(merge-node-configurations (:nodes opts) (:node-configs opts))
|
||||
[(resolve-all-node-hostnames (default-node-configuration (:nodes opts)))])
|
||||
(throw (Exception. "Node config is missing")))
|
||||
test-opts (for [i counts c node-configs w workloads]
|
||||
(assoc opts
|
||||
:node-config c
|
||||
|
||||
Loading…
Reference in New Issue
Block a user