Add Jepsen initialization (#53)

Co-authored-by: Antonio Andelic <antonio.andelic@memgraph.io>

.github/workflows/diff.yaml

@@ -320,3 +320,47 @@ jobs:
         with:
           name: "Enterprise DEB package"
           path: build/output/memgraph*.deb
+        
+  release_jepsen_test:
+    name: "Release Jepsen Test"
+    runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build only memgraph release binarie.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release ..
+          make -j$THREADS memgraph
+
+      - name: Run Jepsen tests
+        run: |
+          cd tests/jepsen
+          ./setup-local-docker-cluster.sh copy ../../build/memgraph
+          # NOTE: docker exec -t is NOT ok because gh user does NOT have TTY.
+          # NOTE: ~/.bashrc has to be manually sourced when bash -c is used
+          #       because some Jepsen config is there.
+          docker exec jepsen-control bash -c "source ~/.bashrc && cd memgraph && lein run test-all --local-binary /opt/memgraph/memgraph --node n1"
+          docker exec jepsen-control bash -c 'tar -czvf /jepsen/memgraph/Jepsen.tar.gz $(readlink -f /jepsen/memgraph/store/latest)'
+          docker cp jepsen-control:/jepsen/memgraph/Jepsen.tar.gz ./
+
+      - name: Save Jepsen report
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Jepsen Report"
+          path: tests/jepsen/Jepsen.tar.gz

.github/workflows/test_all_workers.yaml

@@ -36,6 +36,8 @@ jobs:
         run: |
           source /opt/toolchain-v2/activate
           ./tools/check-build-system
+          docker --version
+          docker ps | grep jepsen
 
   HP-DL360G6-v2-1:
     name: "HP-DL360G6-v2-1"

tests/jepsen/.gitignore

@@ -0,0 +1,13 @@
+/target
+/classes
+/checkouts
+profiles.clj
+pom.xml
+pom.xml.asc
+*.jar
+*.class
+/.lein-*
+/.nrepl-port
+/store
+/jepsen
+/Jepsen.tar.gz

tests/jepsen/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Change Log
+
+All notable changes to this project will be documented in this file. This
+change log follows the conventions of
+[keepachangelog.com](http://keepachangelog.com/).

tests/jepsen/README.md

@@ -0,0 +1,4 @@
+# Jepsen Memgraph Test Suite
+
+NOTE: Jepsen can only connect to the SSH server on the default 22 port.
+`--node` flag only takes the actual address (:port doesn't work).

tests/jepsen/doc/intro.md

@@ -0,0 +1,3 @@
+# Introduction to Jepsen Memgraph Test Suite
+
+TODO: write [great documentation](http://jacobian.org/writing/what-to-write/)

tests/jepsen/project.clj

@@ -0,0 +1,11 @@
+(defproject jepsen.memgraph "0.1.0-SNAPSHOT"
+  :description "A Jepsen test for Memgraph"
+  :url "https://memgraph.com/"
+  :license {:name "Memgraph Enterprise"
+            :url "https://github.com/memgraph/memgraph/blob/master/release/LICENSE_ENTERPRISE.md"}
+  :main jepsen.memgraph.core
+  :dependencies [[org.clojure/clojure "1.10.0"]
+                 [jepsen "0.2.1-SNAPSHOT"]
+                 [gorillalabs/neo4j-clj "4.1.0"]]
+  :profiles {:test {:dependencies [#_[org.neo4j.test/neo4j-harness "4.1.0"]]}}
+  :repl-options {:init-ns jepsen.memgraph.core})

tests/jepsen/setup-local-docker-cluster.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# NOTE: docker and docker-compose have to be installed on the system.
+
+if [ ! -d "$script_dir/jepsen" ]; then
+    git clone https://github.com/jepsen-io/jepsen.git -b "0.2.1" "$script_dir/jepsen"
+fi
+
+# Initialize testing context by copying source/binary files. Inside CI,
+# Memgraph is tested on a single machine cluster based on Docker containers.
+# Once these tests will be part of the official Jepsen repo, the majority of
+# functionalities inside this script won't be needed because each node clones
+# the public repo.
+case $1 in
+    # Start Jepsen Docker cluster.
+    up)
+        "$script_dir/jepsen/docker/bin/up" --daemon
+    ;;
+    # Copy Memgraph Jepsen project files and Memgraph binary if specified.
+    copy)
+        shift 1
+        binary_path=$1
+        if [ -L "$binary_path" ]; then
+            binary_path=$(readlink "$binary_path")
+        fi
+        binary_name=$(basename -- "$binary_path")
+
+        # Copy Memgraph build binary.
+        docker exec jepsen-n1 mkdir -p /opt/memgraph
+        docker exec jepsen-n2 mkdir -p /opt/memgraph
+        docker exec jepsen-n3 mkdir -p /opt/memgraph
+        docker exec jepsen-n4 mkdir -p /opt/memgraph
+        docker exec jepsen-n5 mkdir -p /opt/memgraph
+        docker cp "$binary_path" jepsen-n1:/opt/memgraph/"$binary_name"
+        docker exec jepsen-n1 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph"
+        docker cp "$binary_path" jepsen-n2:/opt/memgraph/"$binary_name"
+        docker exec jepsen-n2 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph"
+        docker cp "$binary_path" jepsen-n3:/opt/memgraph/"$binary_name"
+        docker exec jepsen-n3 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph"
+        docker cp "$binary_path" jepsen-n4:/opt/memgraph/"$binary_name"
+        docker exec jepsen-n4 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph"
+        docker cp "$binary_path" jepsen-n5:/opt/memgraph/"$binary_name"
+        docker exec jepsen-n5 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph"
+
+        # Copy tests/jepsen/memgraph required files into the control node.
+        docker exec jepsen-control mkdir -p /jepsen/memgraph
+        docker cp "$script_dir/src/." jepsen-control:/jepsen/memgraph/src/
+        docker cp "$script_dir/test/." jepsen-control:/jepsen/memgraph/test/
+        docker cp "$script_dir/project.clj" jepsen-control:/jepsen/memgraph/project.clj
+    ;;
+esac

tests/jepsen/src/jepsen/memgraph/basic.clj

@@ -0,0 +1,75 @@
+(ns jepsen.memgraph.basic
+  "Basic Memgraph test"
+  (:require [neo4j-clj.core :as dbclient] 
+            [jepsen [client :as client]
+                    [checker :as checker]
+                    [generator :as gen]]
+            [jepsen.checker.timeline :as timeline]
+            [knossos.model :as model]
+            [slingshot.slingshot :refer [try+ throw+]]
+            [jepsen.memgraph.client :as c]))
+
+;; Operations specific to the tested system (Bolt and Cypher).
+
+(dbclient/defquery create-node
+  "CREATE (n:Node {id: $id, value: $value});")
+(dbclient/defquery get-node
+  "MATCH (n:Node {id: $id}) RETURN n;")
+(dbclient/defquery get-all-nodes
+  "MATCH (n:Node) RETURN n;")
+(dbclient/defquery update-node
+  "MATCH (n:Node {id: $id}) SET n.value = $value;")
+
+(defn compare-and-set-node
+  [conn o n]
+  (dbclient/with-transaction conn tx
+    (if (= (-> (get-node tx {:id "0"}) first :n :value) (str o))
+      (update-node tx {:id "0" :value n})
+      (throw+ "Unable to alter something that does NOT exist."))))
+(dbclient/defquery detach-delete-all
+  "MATCH (n) DETACH DELETE n;")
+
+;; Client specific to the tested system (Bolt and Cypher).
+;; TODO (gitbuda): Write an impl targeting Memgraph replicated cluster.
+(defrecord Client [conn]
+  client/Client
+  (open! [this test node]
+    (assoc this :conn (c/open node)))
+  (setup! [this test]
+    (c/with-session conn session
+      (create-node session {:id "0" :value 0})))
+  (invoke! [this test op]
+    (case (:f op)
+      :read (assoc op :type :ok
+                   :value (c/with-session conn session
+                            (-> (get-node session {:id "0"}) first :n :value)))
+      :write (do (c/with-session conn session
+                   (update-node session {:id "0" :value (:value op)}))
+                 (assoc op :type :ok))
+      :cas (try+
+            (let [[o n] (:value op)]
+              (assoc op :type (if (compare-and-set-node conn o n) :ok :fail)))
+            (catch Object _
+              (assoc op :type :fail, :error :not-found)))))
+  (teardown! [this test]
+    (c/with-session conn session
+      (detach-delete-all session)))
+  (close! [_ est]
+    (dbclient/disconnect conn)))
+
+;; Abstract operations executed against the tested system.
+(defn r [_ _]   {:type :invoke, :f :read,  :value nil})
+(defn w [_ _]   {:type :invoke, :f :write, :value (rand-int 5)})
+(defn cas [_ _] {:type :invoke, :f :cas,   :value [(rand-int 5) (rand-int 5)]})
+
+(defn workload
+  "Basic test workload"
+  [opts]
+  {:client (Client. nil)
+   :checker (checker/compose
+              {:linear (checker/linearizable
+                {:model (model/cas-register 0)
+                 :algorithm :linear})
+               :timeline (timeline/html)})
+   :generator (gen/mix [r w cas])})
+

tests/jepsen/src/jepsen/memgraph/client.clj

@@ -0,0 +1,23 @@
+(ns jepsen.memgraph.client
+  "Neo4j Clojure driver helper functions/macros"
+  (:require [neo4j-clj.core :as dbclient])
+  (:import (java.net URI)))
+
+;; Jepsen related utils.
+(defn instance-url
+  "An URL for connecting to an instance on a particular port"
+  [node port]
+  (str "bolt://" node ":" port))
+
+;; neo4j-clj related utils.
+(defmacro with-session
+  "Execute body expressions by using the same session. Useful when executing
+  multiple queries, each as a separete transaction."
+  [connection session & body]
+  `(with-open [~session (dbclient/get-session ~connection)]
+     ~@body))
+
+(defn open
+  "Open client connection to the node"
+  [node]
+  (dbclient/connect (URI. (instance-url node 7687)) "" ""))

tests/jepsen/src/jepsen/memgraph/core.clj

@@ -0,0 +1,78 @@
+(ns jepsen.memgraph.core
+  (:gen-class)
+  (:require [clojure.tools.logging :refer :all]
+            [clojure.string :as str]
+            [jepsen [cli :as cli]
+                    [core :as jepsen]
+                    [checker :as checker]
+                    [nemesis :as nemesis]
+                    [generator :as gen]
+                    [tests :as tests]]
+            [slingshot.slingshot :refer [try+ throw+]]
+            [jepsen.memgraph [basic :as basic]
+                             [support :as s]]))
+
+(def workloads
+  "A map of workload names to functions that can take opts and construct
+   workloads."
+  {:basic basic/workload})
+
+(defn memgraph-test
+  "Given an options map from the command line runner (e.g. :nodes, :ssh,
+  :concurrency, ...), constructs a test map."
+  [opts]
+  (let [workload ((get workloads (:workload opts)) opts)]
+    (merge tests/noop-test
+           opts
+           {:pure-generators true
+            :name            (str "test-" (name (:workload opts)))
+            :db              (s/db (:package-url opts) (:local-binary opts))
+            :client          (:client workload)
+            :checker         (checker/compose
+                               ;; Fails on a cluster of independent Memgraphs.
+                               {;; :stats      (checker/stats) CAS always fails
+                                ;;                             so enable this
+                                ;;                             if all test have
+                                ;;                             at least 1 ok op
+                                :exceptions (checker/unhandled-exceptions)
+                                :perf       (checker/perf)
+                                :workload   (:checker workload)})
+            :nemesis         (nemesis/partition-random-halves)
+            :generator       (->> (:generator workload)
+                                  (gen/nemesis
+                                   (cycle [(gen/sleep 5)
+                                           {:type :info, :f :start}
+                                           (gen/sleep 5)
+                                           {:type :info, :f :stop}]))
+                                  (gen/time-limit (:time-limit opts)))})))
+
+(def cli-opts
+  "CLI options for tests."
+  [[nil "--package-url URL" "What package of Memgraph should we test?"
+    :default nil]
+   [nil "--local-binary PATH" "Ignore package; use this local binary instead."
+    :default "/opt/memgraph/memgraph"]
+   ["-w" "--workload NAME" "Test workload to run"
+    :parse-fn keyword
+    :validate [workloads (cli/one-of workloads)]]])
+
+(defn all-tests
+  "Takes base CLI options and constructs a sequence of test options."
+  [opts]
+  (let [counts    (range (:test-count opts))
+        workloads (if-let [w (:workload opts)] [w] (keys workloads))
+        test-opts (for [i counts, w workloads]
+                   (assoc opts
+                          :workload w))]
+    (map memgraph-test test-opts)))
+
+(defn -main
+  "Handles command line arguments. Can either run a test, or a web server for
+  browsing results."
+  [& args]
+  (cli/run! (merge (cli/test-all-cmd {:tests-fn all-tests
+                                      :opt-spec cli-opts})
+                   (cli/single-test-cmd {:test-fn memgraph-test
+                                         :opt-spec cli-opts})
+                   (cli/serve-cmd))
+            args))

tests/jepsen/src/jepsen/memgraph/support.clj

@@ -0,0 +1,44 @@
+(ns jepsen.memgraph.support
+  (:require [clojure.string :as str]
+            [clojure.tools.logging :refer [info]]
+            [jepsen [db :as db]
+                    [control :as c]
+                    [util :as util :refer [meh]]]
+            [jepsen.control.util :as cu]
+            [jepsen.os.debian :as debian]))
+
+;; Memgraph database config and setup.
+(def mgdir  "/opt/memgraph")
+(def mgdata (str mgdir "/mg_data"))
+(def mglog  (str mgdir "/memgraph.log"))
+(def mgpid  (str mgdir "/memgraph.pid"))
+(defn db
+  "Manage Memgraph DB on each node."
+  [package-url local-binary]
+  (reify db/DB
+    (setup! [_ test node]
+      (c/su (debian/install ['python3 'python3-dev]))
+      (c/su (meh (c/exec :killall :memgraph)))
+      (when-not (nil? package-url)
+        (throw (Exception. "Memgraph package-url setup not yet implemented.")))
+      (when (nil? local-binary)
+        (throw (Exception. "Memgraph local-binary has to be defined.")))
+      (try (c/exec :command :-v local-binary)
+           (catch Exception e
+             (throw (Exception. (str local-binary " is not there.")))))
+      (info node "Memgraph binary is there" local-binary)
+      (cu/start-daemon!
+       {:logfile mglog
+        :pidfile mgpid
+        :chdir   mgdir}
+       local-binary)
+      (Thread/sleep 2000))
+    (teardown! [_ test node]
+      (info node "Tearing down Memgraph")
+      (when (and local-binary mgpid) (cu/stop-daemon! local-binary mgpid))
+      (c/su
+       (c/exec :rm :-rf mgdata)
+       (c/exec :rm :-rf mglog)))
+    db/LogFiles
+    (log-files [_ test node]
+      [mglog])))

tests/jepsen/test/jepsen/memgraph_test.clj

@@ -0,0 +1,7 @@
+(ns jepsen.memgraph-test
+  (:require [clojure.test :refer :all]
+            [jepsen.memgraph :refer :all]))
+
+(deftest a-test
+  (testing "FIXME, I fail."
+    (is (= 0 1))))