memgraph/tests/jepsen/run.sh

#!/bin/bash

set -Eeuo pipefail
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

HELP_EXIT() {
    echo ""
    echo "HELP: $0 help|cluster-up|test [args]"
    echo ""
    echo "    test args --binary                 MEMGRAPH_BINARY_PATH"
    echo "              --ignore-run-stdout-logs Ignore lein run stdout logs."
    echo "              --ignore-run-stderr-logs Ignore lein run stderr logs."
    echo "              --nodes-no               JEPSEN_ACTIVE_NODES_NO"
    echo "              --run-args               \"CONTROL_LEIN_RUN_ARGS\" (NOTE: quotes)"
    echo ""
    exit 1
}

ERROR() {
    /bin/echo -e "\e[101m\e[97m[ERROR]\e[49m\e[39m" "$@"
}

INFO() {
    /bin/echo -e "\e[104m\e[97m[INFO]\e[49m\e[39m" "$@"
}

if ! command -v docker > /dev/null 2>&1 || ! command -v docker-compose > /dev/null 2>&1; then
  ERROR "docker and docker-compose have to be installed."
  exit 1
fi

MEMGRAPH_BINARY_PATH="../../build/memgraph"
JEPSEN_ACTIVE_NODES_NO=5
CONTROL_LEIN_RUN_ARGS="test-all --node-configs resources/node-config.edn"
CONTROL_LEIN_RUN_STDOUT_LOGS=1
CONTROL_LEIN_RUN_STDERR_LOGS=1

if [ ! -d "$script_dir/jepsen" ]; then
    git clone https://github.com/jepsen-io/jepsen.git -b "0.2.1" "$script_dir/jepsen"
fi

if [ "$#" -lt 1 ]; then
    HELP_EXIT
fi

# Initialize testing context by copying source/binary files. Inside CI,
# Memgraph is tested on a single machine cluster based on Docker containers.
# Once these tests will be part of the official Jepsen repo, the majority of
# functionalities inside this script won't be needed because each node clones
# the public repo.
case $1 in
    help)
        HELP_EXIT
    ;;
    # Start Jepsen Docker cluster of 5 nodes. To configure the cluster please
    # take a look under jepsen/docker/docker-compose.yml.
    # NOTE: If you delete the jepsen folder where docker config is located,
    # the current cluster is broken because it relies on the folder. That can
    # happen easiliy because the jepsen folder is git ignored.
    cluster-up)
        "$script_dir/jepsen/docker/bin/up" --daemon
    ;;
    # Run tests against the specified Memgraph binary.
    test)
        shift
        while [[ $# -gt 0 ]]; do
            key="$1"
            case $key in
                --binary)
                    shift
                    MEMGRAPH_BINARY_PATH="$1"
                    shift
                ;;
                --ignore-run-stdout-logs)
                    CONTROL_LEIN_RUN_STDOUT_LOGS=0
                    shift
                ;;
                --ignore-run-stderr-logs)
                    CONTROL_LEIN_RUN_STDERR_LOGS=0
                    shift
                ;;
                --nodes-no)
                    shift
                    JEPSEN_ACTIVE_NODES_NO="$1"
                    shift
                ;;
                --run-args)
                    shift
                    CONTROL_LEIN_RUN_ARGS="$1"
                    shift
                ;;
                *)
                    ERROR "Unknown option $1."
                    HELP_EXIT
                ;;
            esac
        done

        # Resolve binary path if it is a link.
        binary_path="$MEMGRAPH_BINARY_PATH"
        if [ -L "$binary_path" ]; then
            binary_path=$(readlink "$binary_path")
        fi
        binary_name=$(basename -- "$binary_path")

        # Copy Memgraph binary.
        for iter in $(seq 1 "$JEPSEN_ACTIVE_NODES_NO"); do
            jepsen_node_name="jepsen-n$iter"
            # Cleanup the node folder with previous binaries.
            docker exec "$jepsen_node_name" rm -rf /opt/memgraph/
            docker exec "$jepsen_node_name" mkdir -p /opt/memgraph
            docker cp "$binary_path" "$jepsen_node_name":/opt/memgraph/"$binary_name"
            docker exec "$jepsen_node_name" bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph"
            INFO "Copying $binary_name to $jepsen_node_name DONE."
        done

        # Copy test files into the control node.
        docker exec jepsen-control mkdir -p /jepsen/memgraph
        docker cp "$script_dir/src/." jepsen-control:/jepsen/memgraph/src/
        docker cp "$script_dir/test/." jepsen-control:/jepsen/memgraph/test/
        docker cp "$script_dir/resources/." jepsen-control:/jepsen/memgraph/resources/
        docker cp "$script_dir/project.clj" jepsen-control:/jepsen/memgraph/project.clj
        INFO "Copying test files to jepsen-control DONE."

        start_time="$(docker exec jepsen-control bash -c 'date -u +"%Y%m%dT%H%M%S"').000Z"

        # Run the test.
        # NOTE: docker exec -t is NOT ok because gh CI user does NOT have TTY.
        # NOTE: ~/.bashrc has to be manually sourced when bash -c is used
        #       because some Jepsen config is there.
        set +e
        if [ "$CONTROL_LEIN_RUN_STDOUT_LOGS" -eq 0 ]; then
            redirect_stdout_logs="/dev/null"
        else
            redirect_stdout_logs="/dev/stdout"
        fi
        if [ "$CONTROL_LEIN_RUN_STDERR_LOGS" -eq 0 ]; then
            redirect_stderr_logs="/dev/null"
        else
            redirect_stderr_logs="/dev/stderr"
        fi
        INFO "Jepsen run in progress... START_TIME: $start_time"
        docker exec jepsen-control bash -c "source ~/.bashrc && cd memgraph && lein run $CONTROL_LEIN_RUN_ARGS" 1> $redirect_stdout_logs 2> $redirect_stderr_logs
        # To be able to archive the run result even if the run fails.
        jepsen_run_exit_status=$?
        end_time="$(docker exec jepsen-control bash -c 'date -u +"%Y%m%dT%H%M%S"').000Z"
        INFO "Jepsen run DONE. END_TIME: $end_time"
        set -e

        # Pack all test workload runs between start and end time.
        all_workloads=$(docker exec jepsen-control bash -c 'ls /jepsen/memgraph/store/' | grep test-)
        all_workload_run_folders=""
        for workload in $all_workloads; do
            for time_folder in $(docker exec jepsen-control bash -c "ls /jepsen/memgraph/store/$workload"); do
                if [[ "$time_folder" == "latest" ]]; then
                    continue
                fi
                # The early continue pattern here is nice because bash doesn't
                # have >= for the string comparison (marginal values).
                if [[ "$time_folder" < "$start_time" ]]; then
                    continue
                fi
                if [[ "$time_folder" > "$end_time" ]]; then
                    continue
                fi
                all_workload_run_folders="$all_workload_run_folders /jepsen/memgraph/store/$workload/$time_folder"
            done
        done
        docker exec jepsen-control bash -c "tar -czvf /jepsen/memgraph/Jepsen.tar.gz $all_workload_run_folders"
        docker cp jepsen-control:/jepsen/memgraph/Jepsen.tar.gz ./
        INFO "Test and results packing DONE."

        # If the run has failed, this script also has to return non-zero status.
        if [ "$jepsen_run_exit_status" -ne 0 ]; then
            exit "$jepsen_run_exit_status"
        fi
    ;;
    *)
    HELP_EXIT
    ;;
esac