Upgrade jepsen (#1594)

* Try with jepsen v0.3.5
* Add a few WIP adjustments
* Add replication restore state on startup flag
* Fix some run.sh scripts issues
* Improve cluster commands
* Run Jepsen on debian-12 with toolchain v5
---------
Co-authored-by: Marko Budiselic <mbudiselicbuda@gmail.com>

.github/workflows/diff.yaml

@@ -475,8 +475,8 @@ jobs:
       THREADS: 24
       MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
       MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-      OS: debian-10
+      OS: debian-12
-      TOOLCHAIN: v4
+      TOOLCHAIN: v5
       ARCH: amd
       BUILD_TYPE: RelWithDebInfo
 

environment/os/run.sh

@@ -5,17 +5,20 @@ IFS=' '
 # NOTE: docker_image_name could be local image build based on release/package images.
 # NOTE: each line has to be under quotes, docker_container_type, script_name and docker_image_name separate with a space.
 # "docker_container_type script_name docker_image_name"
+#     docker_container_type OPTIONS:
+#         * mgrun -> running plain/empty operating system for the purposes of testing native memgraph package
+#         * mgbuild -> running the builder container to build memgraph inside it -> it's possible create builder images using release/package/run.sh
 OPERATING_SYSTEMS=(
-  "mgrun amzn-2 amazonlinux:2"
+  # "mgrun amzn-2 amazonlinux:2"
-  "mgrun centos-7 centos:7"
+  # "mgrun centos-7 centos:7"
-  "mgrun centos-9 dokken/centos-stream-9"
+  # "mgrun centos-9 dokken/centos-stream-9"
-  "mgrun debian-10 debian:10"
+  # "mgrun debian-10 debian:10"
-  "mgrun debian-11 debian:11"
+  # "mgrun debian-11 debian:11"
-  "mgrun fedora-36 fedora:36"
+  # "mgrun fedora-36 fedora:36"
-  "mgrun ubuntu-18.04 ubuntu:18.04"
+  # "mgrun ubuntu-18.04 ubuntu:18.04"
-  "mgrun ubuntu-20.04 ubuntu:20.04"
+  # "mgrun ubuntu-20.04 ubuntu:20.04"
-  "mgrun ubuntu-22.04 ubuntu:22.04"
+  # "mgrun ubuntu-22.04 ubuntu:22.04"
-  # "mgbuild centos-7 package-mgbuild_centos-7"
+  # "mgbuild debian-12 memgraph/memgraph-builder:v5_debian-12"
 )
 
 if [ ! "$(docker info)" ]; then
@@ -33,15 +36,25 @@ print_help () {
 # NOTE: This is an idempotent operation!
 # TODO(gitbuda): Consider making docker_run always delete + start a new container or add a new function.
 docker_run () {
-  cnt_name="$1"
+  cnt_type="$1"
-  cnt_image="$2"
+  if [[ "$cnt_type" != "mgbuild" && "$cnt_type" != "mgrun" ]]; then
+    echo "ERROR: Wrong docker_container_type -> valid options are mgbuild, mgrun"
+    exit 1
+  fi
+  cnt_name="$2"
+  cnt_image="$3"
   if [ ! "$(docker ps -q -f name=$cnt_name)" ]; then
       if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
           echo "Cleanup of the old exited container..."
           docker rm $cnt_name
       fi
+      if [[ "$cnt_type" == "mgbuild" ]]; then
+        docker run -d --volume "$SCRIPT_DIR/../../:/memgraph" --network host --name "$cnt_name" "$cnt_image"
+      fi
+      if [[ "$cnt_type" == "mgrun" ]]; then
         docker run -d --volume "$SCRIPT_DIR/../../:/memgraph" --network host --name "$cnt_name" "$cnt_image" sleep infinity
       fi
+  fi
   echo "The $cnt_image container is active under $cnt_name name!"
 }
 
@@ -55,10 +68,10 @@ docker_stop_and_rm () {
   cnt_name="$1"
   if [ "$(docker ps -q -f name=$cnt_name)" ]; then
       docker stop "$1"
+  fi
   if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
     docker rm "$1"
   fi
-  fi
 }
 
 # TODO(gitbuda): Make the call to `install NEW_DEPS` configurable, the question what else is useful?
@@ -71,7 +84,7 @@ start_all () {
     docker_name="${docker_container_type}_$script_name"
     echo ""
     echo "~~~~ OPERATING ON $docker_image as $docker_name..."
-    docker_run "$docker_name" "$docker_image"
+    docker_run "$docker_container_type" "$docker_name" "$docker_image"
     docker_exec "$docker_name" "/memgraph/environment/os/$script_name.sh install NEW_DEPS"
     echo "---- DONE EVERYHING FOR $docker_image as $docker_name..."
     echo ""

tests/jepsen/jepsen_0.3.0.patch

@@ -1,13 +0,0 @@
-diff --git a/docker/control/Dockerfile b/docker/control/Dockerfile
-index 6b2d3c0e..195a7a60 100644
---- a/docker/control/Dockerfile
-+++ b/docker/control/Dockerfile
-@@ -7,7 +7,7 @@ ENV LEIN_ROOT true
- # Jepsen dependencies
- #
- RUN apt-get -y -q update && \
--    apt-get install -qy openjdk-17-jdk-headless \
-+    apt-get install -qy ca-certificates-java openjdk-17-jdk-headless \
-     libjna-java \
-     vim \
-     emacs \

tests/jepsen/project.clj

@@ -5,10 +5,10 @@
             :url "https://github.com/memgraph/memgraph/blob/master/release/LICENSE_ENTERPRISE.md"}
   :main jepsen.memgraph.core
   :dependencies [[org.clojure/clojure "1.10.0"]
-                 ;; 0.2.4-SNAPSHOT but 0.3.0, for more -> https://clojars.org/jepsen/versions
+                 ;; Details under https://clojars.org/jepsen/versions.
-                 [jepsen "0.2.4-SNAPSHOT"]
+                 [jepsen "0.3.5-SNAPSHOT"]
                  [gorillalabs/neo4j-clj "4.1.0"]]
   :profiles {:test {:dependencies [#_[org.neo4j.test/neo4j-harness "4.1.0"]]}}
-  ;; required to run 0.3.0
+  ;; The below line is required to run after Jepsen 0.3.0.
-  ; :aot :all
+  :aot :all
   :repl-options {:init-ns jepsen.memgraph.core})

tests/jepsen/run.sh

@@ -2,12 +2,10 @@
 set -Eeuo pipefail
 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-MEMGRAPH_BINARY_PATH="../../build/memgraph"
+MEMGRAPH_BUILD_PATH="$script_dir/../../build"
+MEMGRAPH_BINARY_PATH="$MEMGRAPH_BUILD_PATH/memgraph"
 # NOTE: Jepsen Git tags are not consistent, there are: 0.2.4, v0.3.0, 0.3.2, ...
-# NOTE: On Ubuntu 22.04 v0.3.2 uses non-existing docker compose --compatibility flag.
+JEPSEN_VERSION="${JEPSEN_VERSION:-v0.3.5}"
-# NOTE: On Ubuntu 22.04 v0.3.0 and v0.3.1 seems to be runnable.
-# TODO(gitbuda): Make sure Memgraph can be testes with Jepsen >= 0.3.0
-JEPSEN_VERSION="${JEPSEN_VERSION:-0.2.4}"
 JEPSEN_ACTIVE_NODES_NO=5
 CONTROL_LEIN_RUN_ARGS="test-all --node-configs resources/node-config.edn"
 CONTROL_LEIN_RUN_STDOUT_LOGS=1
@@ -24,7 +22,7 @@ PRINT_CONTEXT() {
 
 HELP_EXIT() {
     echo ""
-    echo "HELP: $0 help|cluster-up|cluster-refresh|cluster-cleanup|cluster-dealloc|mgbuild|test|test-all-individually [args]"
+    echo "HELP: $0 help|cluster-up|cluster-refresh|cluster-nodes-cleanup|cluster-dealloc|mgbuild|test|test-all-individually [args]"
     echo ""
     echo "    test args --binary                 MEMGRAPH_BINARY_PATH"
     echo "              --ignore-run-stdout-logs Ignore lein run stdout logs."
@@ -43,24 +41,18 @@ INFO() {
     /bin/echo -e "\e[104m\e[97m[INFO]\e[49m\e[39m" "$@"
 }
 
+if [[ "$#" -lt 1 || "$1" == "-h" || "$1" == "--help" ]]; then
+    HELP_EXIT
+fi
+
 if ! command -v docker > /dev/null 2>&1 || ! command -v docker-compose > /dev/null 2>&1; then
   ERROR "docker and docker-compose have to be installed."
   exit 1
 fi
 
 if [ ! -d "$script_dir/jepsen" ]; then
+    # TODO(deda): install apt get docker-compose-plugin on all build machines.
     git clone https://github.com/jepsen-io/jepsen.git -b "$JEPSEN_VERSION" "$script_dir/jepsen"
-    if [ "$JEPSEN_VERSION" == "v0.3.0" ]; then
-        if [ -f "$script_dir/jepsen_0.3.0.patch" ]; then
-            cd "$script_dir/jepsen"
-            git apply "$script_dir/jepsen_0.3.0.patch"
-            cd "$script_dir"
-        fi
-    fi
-fi
-
-if [ "$#" -lt 1 ]; then
-    HELP_EXIT
 fi
 
 PROCESS_ARGS() {
@@ -199,7 +191,7 @@ CLUSTER_UP() {
 }
 
 CLUSTER_DEALLOC() {
-  ps=$(docker ps --filter name=jepsen* --filter status=running -q)
+  ps=$(docker ps -a --filter name=jepsen* -q)
   if [[ ! -z ${ps} ]]; then
       echo "Killing ${ps}"
       docker rm -f ${ps}
@@ -213,6 +205,7 @@ CLUSTER_DEALLOC() {
   else
       echo "No Jepsen containers detected!"
   fi
+  echo "Cluster dealloc DONE"
 }
 
 # Initialize testing context by copying source/binary files. Inside CI,
@@ -239,7 +232,7 @@ case $1 in
         CLUSTER_DEALLOC
     ;;
 
-    cluster-cleanup)
+    cluster-nodes-cleanup)
         jepsen_control_exec="docker exec jepsen-control bash -c"
         INFO "Deleting /jepsen/memgraph/store/* on jepsen-control"
         $jepsen_control_exec "rm -rf /jepsen/memgraph/store/*"
@@ -252,9 +245,13 @@ case $1 in
     ;;
 
     mgbuild)
+        PROCESS_ARGS "$@"
         PRINT_CONTEXT
+        # docker cp -L mgbuild_debian-12:/memgraph/build/memgraph "${MEMGRAPH_BUILD_PATH}/"
+        # NOTE: mgconsole is interesting inside jepsen container to inspect Memgraph state.
+        # docker cp -L mgbuild_debian-12:/usr/local/bin/mgconsole "${MEMGRAPH_BUILD_PATH}/"
         echo ""
-        echo "TODO(gitbuda): Build memgraph for Debian 10 via memgraph/memgraph-builder"
+        echo "TODO(gitbuda): Build memgraph for Jepsen (on v0.3.5 for Debian 12) via memgraph/memgraph-builder"
         exit 1
     ;;
 

tests/jepsen/src/jepsen/memgraph/support.clj

@@ -25,6 +25,7 @@
    :--storage-recover-on-startup
    :--storage-wal-enabled
    :--storage-snapshot-interval-sec 300
+   :--replication-restore-state-on-startup
    :--storage-properties-on-edges))
 
 (defn stop-node!