tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Upgrade jepsen (#1594)

Browse Source 
* Try with jepsen v0.3.5
* Add a few WIP adjustments
* Add replication restore state on startup flag
* Fix some run.sh scripts issues
* Improve cluster commands
* Run Jepsen on debian-12 with toolchain v5
---------
Co-authored-by: Marko Budiselic <mbudiselicbuda@gmail.com>
This commit is contained in:
Marko Barišić 2024-03-18 16:38:58 +01:00 committed by GitHub
parent ec8536e11b
commit 2ac649f3b5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 54 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/diff.yaml vendored
View File

@ -475,8 +475,8 @@ jobs:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
OS: debian-10
TOOLCHAIN: v4
OS: debian-12
TOOLCHAIN: v5
ARCH: amd
BUILD_TYPE: RelWithDebInfo

47
environment/os/run.sh
View File

@ -5,17 +5,20 @@ IFS=' '
# NOTE: docker_image_name could be local image build based on release/package images.
# NOTE: each line has to be under quotes, docker_container_type, script_name and docker_image_name separate with a space.
# "docker_container_type script_name docker_image_name"
# docker_container_type OPTIONS:
# * mgrun -> running plain/empty operating system for the purposes of testing native memgraph package
# * mgbuild -> running the builder container to build memgraph inside it -> it's possible create builder images using release/package/run.sh
OPERATING_SYSTEMS=(
"mgrun amzn-2 amazonlinux:2"
"mgrun centos-7 centos:7"
"mgrun centos-9 dokken/centos-stream-9"
"mgrun debian-10 debian:10"
"mgrun debian-11 debian:11"
"mgrun fedora-36 fedora:36"
"mgrun ubuntu-18.04 ubuntu:18.04"
"mgrun ubuntu-20.04 ubuntu:20.04"
"mgrun ubuntu-22.04 ubuntu:22.04"
# "mgbuild centos-7 package-mgbuild_centos-7"
# "mgrun amzn-2 amazonlinux:2"
# "mgrun centos-7 centos:7"
# "mgrun centos-9 dokken/centos-stream-9"
# "mgrun debian-10 debian:10"
# "mgrun debian-11 debian:11"
# "mgrun fedora-36 fedora:36"
# "mgrun ubuntu-18.04 ubuntu:18.04"
# "mgrun ubuntu-20.04 ubuntu:20.04"
# "mgrun ubuntu-22.04 ubuntu:22.04"
# "mgbuild debian-12 memgraph/memgraph-builder:v5_debian-12"
)
if [ ! "$(docker info)" ]; then
@ -33,14 +36,24 @@ print_help () {
# NOTE: This is an idempotent operation!
# TODO(gitbuda): Consider making docker_run always delete + start a new container or add a new function.
docker_run () {
cnt_name="$1"
cnt_image="$2"
cnt_type="$1"
if [[ "$cnt_type" != "mgbuild" && "$cnt_type" != "mgrun" ]]; then
echo "ERROR: Wrong docker_container_type -> valid options are mgbuild, mgrun"
exit 1
fi
cnt_name="$2"
cnt_image="$3"
if [ ! "$(docker ps -q -f name=$cnt_name)" ]; then
if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
echo "Cleanup of the old exited container..."
docker rm $cnt_name
fi
docker run -d --volume "$SCRIPT_DIR/../../:/memgraph" --network host --name "$cnt_name" "$cnt_image" sleep infinity
if [[ "$cnt_type" == "mgbuild" ]]; then
docker run -d --volume "$SCRIPT_DIR/../../:/memgraph" --network host --name "$cnt_name" "$cnt_image"
fi
if [[ "$cnt_type" == "mgrun" ]]; then
docker run -d --volume "$SCRIPT_DIR/../../:/memgraph" --network host --name "$cnt_name" "$cnt_image" sleep infinity
fi
fi
echo "The $cnt_image container is active under $cnt_name name!"
}
@ -55,9 +68,9 @@ docker_stop_and_rm () {
cnt_name="$1"
if [ "$(docker ps -q -f name=$cnt_name)" ]; then
docker stop "$1"
if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
docker rm "$1"
fi
fi
if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
docker rm "$1"
fi
}
@ -71,7 +84,7 @@ start_all () {
docker_name="${docker_container_type}_$script_name"
echo ""
echo "~~~~ OPERATING ON $docker_image as $docker_name..."
docker_run "$docker_name" "$docker_image"
docker_run "$docker_container_type" "$docker_name" "$docker_image"
docker_exec "$docker_name" "/memgraph/environment/os/$script_name.sh install NEW_DEPS"
echo "---- DONE EVERYHING FOR $docker_image as $docker_name..."
echo ""

13
tests/jepsen/jepsen_0.3.0.patch
View File

@ -1,13 +0,0 @@
diff --git a/docker/control/Dockerfile b/docker/control/Dockerfile
index 6b2d3c0e..195a7a60 100644
--- a/docker/control/Dockerfile
+++ b/docker/control/Dockerfile
@@ -7,7 +7,7 @@ ENV LEIN_ROOT true
# Jepsen dependencies
#
RUN apt-get -y -q update && \
- apt-get install -qy openjdk-17-jdk-headless \
+ apt-get install -qy ca-certificates-java openjdk-17-jdk-headless \
libjna-java \
vim \
emacs \

8
tests/jepsen/project.clj
View File

@ -5,10 +5,10 @@
:url "https://github.com/memgraph/memgraph/blob/master/release/LICENSE_ENTERPRISE.md"}
:main jepsen.memgraph.core
:dependencies [[org.clojure/clojure "1.10.0"]
;; 0.2.4-SNAPSHOT but 0.3.0, for more -> https://clojars.org/jepsen/versions
[jepsen "0.2.4-SNAPSHOT"]
;; Details under https://clojars.org/jepsen/versions.
[jepsen "0.3.5-SNAPSHOT"]
[gorillalabs/neo4j-clj "4.1.0"]]
:profiles {:test {:dependencies [#_[org.neo4j.test/neo4j-harness "4.1.0"]]}}
;; required to run 0.3.0
; :aot :all
;; The below line is required to run after Jepsen 0.3.0.
:aot :all
:repl-options {:init-ns jepsen.memgraph.core})

37
tests/jepsen/run.sh
View File

@ -2,12 +2,10 @@
set -Eeuo pipefail
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
MEMGRAPH_BINARY_PATH="../../build/memgraph"
MEMGRAPH_BUILD_PATH="$script_dir/../../build"
MEMGRAPH_BINARY_PATH="$MEMGRAPH_BUILD_PATH/memgraph"
# NOTE: Jepsen Git tags are not consistent, there are: 0.2.4, v0.3.0, 0.3.2, ...
# NOTE: On Ubuntu 22.04 v0.3.2 uses non-existing docker compose --compatibility flag.
# NOTE: On Ubuntu 22.04 v0.3.0 and v0.3.1 seems to be runnable.
# TODO(gitbuda): Make sure Memgraph can be testes with Jepsen >= 0.3.0
JEPSEN_VERSION="${JEPSEN_VERSION:-0.2.4}"
JEPSEN_VERSION="${JEPSEN_VERSION:-v0.3.5}"
JEPSEN_ACTIVE_NODES_NO=5
CONTROL_LEIN_RUN_ARGS="test-all --node-configs resources/node-config.edn"
CONTROL_LEIN_RUN_STDOUT_LOGS=1
@ -24,7 +22,7 @@ PRINT_CONTEXT() {
HELP_EXIT() {
echo ""
echo "HELP: $0 help|cluster-up|cluster-refresh|cluster-cleanup|cluster-dealloc|mgbuild|test|test-all-individually [args]"
echo "HELP: $0 help|cluster-up|cluster-refresh|cluster-nodes-cleanup|cluster-dealloc|mgbuild|test|test-all-individually [args]"
echo ""
echo " test args --binary MEMGRAPH_BINARY_PATH"
echo " --ignore-run-stdout-logs Ignore lein run stdout logs."
@ -43,24 +41,18 @@ INFO() {
/bin/echo -e "\e[104m\e[97m[INFO]\e[49m\e[39m" "$@"
}
if [[ "$#" -lt 1 || "$1" == "-h" || "$1" == "--help" ]]; then
HELP_EXIT
fi
if ! command -v docker > /dev/null 2>&1 || ! command -v docker-compose > /dev/null 2>&1; then
ERROR "docker and docker-compose have to be installed."
exit 1
fi
if [ ! -d "$script_dir/jepsen" ]; then
# TODO(deda): install apt get docker-compose-plugin on all build machines.
git clone https://github.com/jepsen-io/jepsen.git -b "$JEPSEN_VERSION" "$script_dir/jepsen"
if [ "$JEPSEN_VERSION" == "v0.3.0" ]; then
if [ -f "$script_dir/jepsen_0.3.0.patch" ]; then
cd "$script_dir/jepsen"
git apply "$script_dir/jepsen_0.3.0.patch"
cd "$script_dir"
fi
fi
fi
if [ "$#" -lt 1 ]; then
HELP_EXIT
fi
PROCESS_ARGS() {
@ -199,7 +191,7 @@ CLUSTER_UP() {
}
CLUSTER_DEALLOC() {
ps=$(docker ps --filter name=jepsen* --filter status=running -q)
ps=$(docker ps -a --filter name=jepsen* -q)
if [[ ! -z ${ps} ]]; then
echo "Killing ${ps}"
docker rm -f ${ps}
@ -213,6 +205,7 @@ CLUSTER_DEALLOC() {
else
echo "No Jepsen containers detected!"
fi
echo "Cluster dealloc DONE"
}
# Initialize testing context by copying source/binary files. Inside CI,
@ -239,7 +232,7 @@ case $1 in
CLUSTER_DEALLOC
;;
cluster-cleanup)
cluster-nodes-cleanup)
jepsen_control_exec="docker exec jepsen-control bash -c"
INFO "Deleting /jepsen/memgraph/store/* on jepsen-control"
$jepsen_control_exec "rm -rf /jepsen/memgraph/store/*"
@ -252,9 +245,13 @@ case $1 in
;;
mgbuild)
PROCESS_ARGS "$@"
PRINT_CONTEXT
# docker cp -L mgbuild_debian-12:/memgraph/build/memgraph "${MEMGRAPH_BUILD_PATH}/"
# NOTE: mgconsole is interesting inside jepsen container to inspect Memgraph state.
# docker cp -L mgbuild_debian-12:/usr/local/bin/mgconsole "${MEMGRAPH_BUILD_PATH}/"
echo ""
echo "TODO(gitbuda): Build memgraph for Debian 10 via memgraph/memgraph-builder"
echo "TODO(gitbuda): Build memgraph for Jepsen (on v0.3.5 for Debian 12) via memgraph/memgraph-builder"
exit 1
;;

1
tests/jepsen/src/jepsen/memgraph/support.clj
View File

@ -25,6 +25,7 @@
:--storage-recover-on-startup
:--storage-wal-enabled
:--storage-snapshot-interval-sec 300
:--replication-restore-state-on-startup
:--storage-properties-on-edges))
(defn stop-node!