57b84f2da3
Summary: In order to add kafka benchmark, `memgraph_bolt.cpp` has been split. Now we have `memgraph_init.cpp/hpp` files with common memgraph startup code. Kafka benchmark implements a new `main` function that doesn't start a bolt server, it just creates and starts a stream. Then it waits for the stream to start consuming and measures the time it took to import the given number of entries. This benchmark is in a new folder, `feature_benchmark`, and so should any new bechmark that measures performance of memgraphs features. Reviewers: mferencevic, teon.banek, ipaljak, vkasljevic Reviewed By: mferencevic, teon.banek Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D1552
144 lines
3.2 KiB
Bash
Executable File
144 lines
3.2 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
## Helper functions
|
|
|
|
function wait_for_server {
|
|
port=$1
|
|
while ! nc -z -w 1 127.0.0.1 $port; do
|
|
sleep 0.1
|
|
done
|
|
sleep 1
|
|
}
|
|
|
|
function echo_info { printf "\033[1;36m~~ $1 ~~\033[0m\n"; }
|
|
function echo_success { printf "\033[1;32m~~ $1 ~~\033[0m\n\n"; }
|
|
function echo_failure { printf "\033[1;31m~~ $1 ~~\033[0m\n\n"; }
|
|
|
|
|
|
## Environment setup
|
|
|
|
# Get script location.
|
|
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
|
cd "$DIR"
|
|
|
|
# Create a temporary directory.
|
|
tmpdir=/tmp/memgraph_benchmark_kafka
|
|
if [ -d $tmpdir ]; then
|
|
rm -rf $tmpdir
|
|
fi
|
|
mkdir -p $tmpdir
|
|
cd $tmpdir
|
|
|
|
# Download the kafka binaries.
|
|
kafka="kafka_2.11-2.0.0"
|
|
wget -nv http://deps.memgraph.io/$kafka.tgz
|
|
tar -xf $kafka.tgz
|
|
mv $kafka kafka
|
|
|
|
# Find memgraph binaries.
|
|
binary_dir="$DIR/../../../build"
|
|
if [ ! -d $binary_dir ]; then
|
|
binary_dir="$DIR/../../../build_release"
|
|
fi
|
|
|
|
# Cleanup old kafka logs.
|
|
if [ -d /tmp/kafka-logs ]; then
|
|
rm -rf /tmp/kafka-logs
|
|
fi
|
|
if [ -d /tmp/zookeeper ]; then
|
|
rm -rf /tmp/zookeeper
|
|
fi
|
|
|
|
# Results for apollo
|
|
RESULTS="$DIR/.apollo_measurements"
|
|
|
|
# Benchmark parameters
|
|
NODES=100000
|
|
EDGES=10000
|
|
|
|
|
|
## Startup
|
|
|
|
# Start the zookeeper process and wait for it to start.
|
|
echo_info "Starting zookeeper"
|
|
./kafka/bin/zookeeper-server-start.sh -daemon kafka/config/zookeeper.properties
|
|
wait_for_server 2181
|
|
echo_success "Started zookeeper"
|
|
|
|
# Start the kafka process and wait for it to start.
|
|
echo_info "Starting kafka"
|
|
./kafka/bin/kafka-server-start.sh -daemon kafka/config/server.properties
|
|
wait_for_server 9092
|
|
echo_success "Started kafka"
|
|
|
|
# Create the kafka topic.
|
|
echo_info "Creating kafka topic test"
|
|
./kafka/bin/kafka-topics.sh --create \
|
|
--zookeeper 127.0.0.1:2181 \
|
|
--replication-factor 1 \
|
|
--partitions 1 \
|
|
--topic test
|
|
echo_success "Created kafka topic test"
|
|
|
|
# Start a http server to serve the transform script.
|
|
echo_info "Starting Python HTTP server"
|
|
mkdir serve
|
|
cd serve
|
|
cp "$DIR/transform.py" transform.py
|
|
python3 -m http.server &
|
|
http_server_pid=$!
|
|
wait_for_server 8000
|
|
cd ..
|
|
echo_success "Started Python HTTP server"
|
|
|
|
|
|
# Start the memgraph process and wait for it to start.
|
|
echo_info "Starting kafka benchmark"
|
|
$binary_dir/tests/feature_benchmark/kafka/benchmark \
|
|
--import-count=$(($NODES + $EDGES)) \
|
|
--timeout=60 \
|
|
--kafka-uri="127.0.0.1:9092" \
|
|
--topic-name="test" \
|
|
--transform-uri="127.0.0.1:8000/transform.py" \
|
|
--output-file=$RESULTS &
|
|
pid=$!
|
|
|
|
# Allow benchmark to initialize
|
|
sleep 5
|
|
|
|
echo_info "Generating kafka messages"
|
|
python3 "$DIR/generate.py" --nodes $NODES --edges $EDGES | \
|
|
./kafka/bin/kafka-console-producer.sh \
|
|
--broker-list 127.0.0.1:9092 \
|
|
--topic test \
|
|
> /dev/null
|
|
echo_success "Finished generating kafka messages"
|
|
|
|
wait -n $pid
|
|
code=$?
|
|
|
|
if [ $code -eq 0 ]; then
|
|
echo_success "Benchmark finished successfully"
|
|
else
|
|
echo_failure "Benchmark didn't finish successfully"
|
|
fi
|
|
|
|
## Cleanup
|
|
|
|
echo_info "Starting test cleanup"
|
|
|
|
# Shutdown the http server.
|
|
kill $http_server_pid
|
|
wait -n
|
|
|
|
# Shutdown the kafka process.
|
|
./kafka/bin/kafka-server-stop.sh
|
|
|
|
# Shutdown the zookeeper process.
|
|
./kafka/bin/zookeeper-server-stop.sh
|
|
|
|
echo_success "Test cleanup done"
|
|
|
|
[ $code -ne 0 ] && exit $code
|
|
exit 0
|