A python script that visualizes latency results from LDBC.
Summary: * extension of run_benchmark script * tools/plot_latency.py Reviewers: teon.banek Reviewed By: teon.banek Subscribers: mislav.bradac, pullbot Differential Revision: https://phabricator.memgraph.io/D743
This commit is contained in:
Marko Budiselic
parent
c56621682e
commit
b30e3252e2
@ -0,0 +1,71 @@
|
||||
# Linked Data Benchmark Council
|
||||
# Social Network Benchmark
|
||||
# Interactive Workload
|
||||
|
||||
# *** workload-related driver properties ***
|
||||
workload=com.ldbc.driver.workloads.ldbc.snb.interactive.LdbcSnbInteractiveWorkload
|
||||
|
||||
# *** vendor-related driver properties ***
|
||||
# database=
|
||||
|
||||
# *** workload-specific properties ***
|
||||
|
||||
# Directory containing query parameter files
|
||||
# ldbc.snb.interactive.parameters_dir=
|
||||
|
||||
# Directory containing data update stream files
|
||||
# ldbc.snb.interactive.updates_dir=
|
||||
|
||||
# Short reads random walk dissipation rate, in the interval [1.0-0.0]
|
||||
# Higher values translate to shorter walks and therefore fewer short reads
|
||||
ldbc.snb.interactive.short_read_dissipation=0.2
|
||||
|
||||
## frequency of read queries (number of update queries per one read query)
|
||||
ldbc.snb.interactive.LdbcQuery1_freq=26
|
||||
ldbc.snb.interactive.LdbcQuery2_freq=37
|
||||
ldbc.snb.interactive.LdbcQuery3_freq=69
|
||||
ldbc.snb.interactive.LdbcQuery4_freq=36
|
||||
ldbc.snb.interactive.LdbcQuery5_freq=57
|
||||
ldbc.snb.interactive.LdbcQuery6_freq=129
|
||||
ldbc.snb.interactive.LdbcQuery7_freq=87
|
||||
ldbc.snb.interactive.LdbcQuery8_freq=45
|
||||
ldbc.snb.interactive.LdbcQuery9_freq=157
|
||||
ldbc.snb.interactive.LdbcQuery10_freq=30
|
||||
ldbc.snb.interactive.LdbcQuery11_freq=16
|
||||
ldbc.snb.interactive.LdbcQuery12_freq=44
|
||||
ldbc.snb.interactive.LdbcQuery13_freq=19
|
||||
ldbc.snb.interactive.LdbcQuery14_freq=49
|
||||
|
||||
# *** For debugging purposes ***
|
||||
|
||||
ldbc.snb.interactive.LdbcQuery1_enable=false
|
||||
ldbc.snb.interactive.LdbcQuery2_enable=false
|
||||
ldbc.snb.interactive.LdbcQuery3_enable=false
|
||||
ldbc.snb.interactive.LdbcQuery4_enable=false
|
||||
ldbc.snb.interactive.LdbcQuery5_enable=false
|
||||
ldbc.snb.interactive.LdbcQuery6_enable=false
|
||||
ldbc.snb.interactive.LdbcQuery7_enable=false
|
||||
ldbc.snb.interactive.LdbcQuery8_enable=true
|
||||
ldbc.snb.interactive.LdbcQuery9_enable=false
|
||||
ldbc.snb.interactive.LdbcQuery10_enable=false
|
||||
ldbc.snb.interactive.LdbcQuery11_enable=false
|
||||
ldbc.snb.interactive.LdbcQuery12_enable=false
|
||||
ldbc.snb.interactive.LdbcQuery13_enable=false
|
||||
ldbc.snb.interactive.LdbcQuery14_enable=false
|
||||
|
||||
ldbc.snb.interactive.LdbcShortQuery1PersonProfile_enable=true
|
||||
ldbc.snb.interactive.LdbcShortQuery2PersonPosts_enable=true
|
||||
ldbc.snb.interactive.LdbcShortQuery3PersonFriends_enable=true
|
||||
ldbc.snb.interactive.LdbcShortQuery4MessageContent_enable=true
|
||||
ldbc.snb.interactive.LdbcShortQuery5MessageCreator_enable=true
|
||||
ldbc.snb.interactive.LdbcShortQuery6MessageForum_enable=true
|
||||
ldbc.snb.interactive.LdbcShortQuery7MessageReplies_enable=true
|
||||
|
||||
ldbc.snb.interactive.LdbcUpdate1AddPerson_enable=true
|
||||
ldbc.snb.interactive.LdbcUpdate2AddPostLike_enable=true
|
||||
ldbc.snb.interactive.LdbcUpdate3AddCommentLike_enable=true
|
||||
ldbc.snb.interactive.LdbcUpdate4AddForum_enable=true
|
||||
ldbc.snb.interactive.LdbcUpdate5AddForumMembership_enable=true
|
||||
ldbc.snb.interactive.LdbcUpdate6AddPost_enable=true
|
||||
ldbc.snb.interactive.LdbcUpdate7AddComment_enable=true
|
||||
ldbc.snb.interactive.LdbcUpdate8AddFriendship_enable=true
|
||||
@ -62,10 +62,10 @@ ldbc.snb.interactive.LdbcShortQuery6MessageForum_enable=true
|
||||
ldbc.snb.interactive.LdbcShortQuery7MessageReplies_enable=true
|
||||
|
||||
ldbc.snb.interactive.LdbcUpdate1AddPerson_enable=false
|
||||
ldbc.snb.interactive.LdbcUpdate2AddPostLike_enable=false
|
||||
ldbc.snb.interactive.LdbcUpdate3AddCommentLike_enable=false
|
||||
ldbc.snb.interactive.LdbcUpdate2AddPostLike_enable=true
|
||||
ldbc.snb.interactive.LdbcUpdate3AddCommentLike_enable=true
|
||||
ldbc.snb.interactive.LdbcUpdate4AddForum_enable=false
|
||||
ldbc.snb.interactive.LdbcUpdate5AddForumMembership_enable=false
|
||||
ldbc.snb.interactive.LdbcUpdate5AddForumMembership_enable=true
|
||||
ldbc.snb.interactive.LdbcUpdate6AddPost_enable=false
|
||||
ldbc.snb.interactive.LdbcUpdate7AddComment_enable=false
|
||||
ldbc.snb.interactive.LdbcUpdate8AddFriendship_enable=false
|
||||
ldbc.snb.interactive.LdbcUpdate8AddFriendship_enable=true
|
||||
|
||||
2
tests/public_benchmark/ldbc/plots/.gitignore
vendored
Normal file
2
tests/public_benchmark/ldbc/plots/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
*
|
||||
!.gitignore
|
||||
2
tests/public_benchmark/ldbc/results/.gitignore
vendored
Normal file
2
tests/public_benchmark/ldbc/results/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
*
|
||||
!.gitignore
|
||||
@ -16,14 +16,16 @@ function print_help () {
|
||||
echo -e " --time-compression-ratio |"
|
||||
echo -e " --operation-count | -> https://github.com/ldbc/ldbc_driver/wiki/Driver-Configuration"
|
||||
echo -e " --thread-count |"
|
||||
echo -e " --result-file-prefix -> Result file prefix."
|
||||
}
|
||||
|
||||
# Default parameters.
|
||||
host=127.0.0.1
|
||||
port=7687
|
||||
time_compression_ratio=0.01
|
||||
operation_count=100
|
||||
operation_count=200
|
||||
thread_count=8
|
||||
result_file_prefix="undefined"
|
||||
|
||||
# Read the arguments.
|
||||
while [[ $# -gt 0 ]]
|
||||
@ -41,6 +43,10 @@ do
|
||||
port=$2
|
||||
shift
|
||||
;;
|
||||
--result-file-prefix)
|
||||
result_file_prefix=$2
|
||||
shift
|
||||
;;
|
||||
--time-compression-ratio)
|
||||
time_compression_ratio=$2
|
||||
shift
|
||||
@ -64,4 +70,6 @@ cd ${script_dir}/ldbc-snb-impls
|
||||
mvn clean compile assembly:single
|
||||
|
||||
cd ${script_dir}/ldbc_driver
|
||||
java -cp target/jeeves-0.3-SNAPSHOT.jar:${script_dir}/ldbc-snb-impls/snb-interactive-neo4j/target/snb-interactive-neo4j-1.0.0-jar-with-dependencies.jar com.ldbc.driver.Client -P ${script_dir}/ldbc_driver/configuration/ldbc_driver_default.properties -P ${script_dir}/ldbc-snb-impls-test.properties -P ${script_dir}/ldbc_snb_datagen/social_network/updateStream.properties -p host ${host} -p port ${port} -db net.ellitron.ldbcsnbimpls.interactive.neo4j.Neo4jDb -p ldbc.snb.interactive.parameters_dir ${script_dir}/ldbc_snb_datagen/substitution_parameters --time_compression_ratio ${time_compression_ratio} --operation_count ${operation_count} --thread_count ${thread_count}
|
||||
java -cp target/jeeves-0.3-SNAPSHOT.jar:${script_dir}/ldbc-snb-impls/snb-interactive-neo4j/target/snb-interactive-neo4j-1.0.0-jar-with-dependencies.jar com.ldbc.driver.Client -P ${script_dir}/ldbc_driver/configuration/ldbc_driver_default.properties -P ${script_dir}/ldbc-snb-impls-allshortandupdate.properties -P ${script_dir}/ldbc_snb_datagen/social_network/updateStream.properties -p host ${host} -p port ${port} -db net.ellitron.ldbcsnbimpls.interactive.neo4j.Neo4jDb -p ldbc.snb.interactive.parameters_dir ${script_dir}/ldbc_snb_datagen/substitution_parameters --time_compression_ratio ${time_compression_ratio} --operation_count ${operation_count} --thread_count ${thread_count}
|
||||
|
||||
cp ${script_dir}/ldbc_driver/results/LDBC-results.json ${script_dir}/results/${result_file_prefix}-LDBC-results.json
|
||||
|
||||
132
tools/plot_ldbc_latency
Executable file
132
tools/plot_ldbc_latency
Executable file
@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
'''
|
||||
Latency Barchart (Based on LDBC JSON output).
|
||||
'''
|
||||
|
||||
import json
|
||||
import os
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.cbook import get_sample_data
|
||||
from argparse import ArgumentParser
|
||||
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
COLORS = {
|
||||
'memgraph': '#ff7300',
|
||||
'neo4j': '#008cc2'
|
||||
}
|
||||
|
||||
|
||||
def parse_args():
|
||||
argp = ArgumentParser(description=__doc__)
|
||||
argp.add_argument("--vendor-references", nargs="+",
|
||||
help="Short references that represent all the "
|
||||
"vendors that are going to be "
|
||||
"visualized on the plot.")
|
||||
argp.add_argument("--vendor-titles", nargs="+",
|
||||
help="Vender titles that are going to appear "
|
||||
"on the plot, e.g. legend titles.")
|
||||
argp.add_argument("--plot-title", default="{{Plot title placeholder}}",
|
||||
help="Plot title.")
|
||||
argp.add_argument("--logo-path", default=None,
|
||||
help="Path to the logo that is going to be presented"
|
||||
" instead of title.")
|
||||
argp.add_argument("--results-dir",
|
||||
default=os.path.join(SCRIPT_DIR,
|
||||
"../tests/public_benchmark"
|
||||
"/ldbc/results"),
|
||||
help="Path to the folder with result files in format "
|
||||
"{{vendor-reference}}-LDBC-results.json")
|
||||
return argp.parse_args()
|
||||
|
||||
|
||||
def autolabel(ax, rects):
|
||||
"""
|
||||
Attach a text label above each bar displaying its height
|
||||
"""
|
||||
for rect in rects:
|
||||
height = rect.get_height()
|
||||
# TODO: adjust more vendors
|
||||
ax.text(rect.get_x() + rect.get_width()/2., 1.00*height,
|
||||
'%d' % int(height),
|
||||
ha='center', va='bottom')
|
||||
|
||||
|
||||
def main():
|
||||
# Read the arguments.
|
||||
args = parse_args()
|
||||
|
||||
# Prepare the datastructure.
|
||||
vendors = {}
|
||||
for vendor_reference, vendor_title in zip(args.vendor_references,
|
||||
args.vendor_titles):
|
||||
vendors[vendor_reference] = {}
|
||||
vendors[vendor_reference]['title'] = vendor_title
|
||||
vendors[vendor_reference]['results_path'] = os.path.join(
|
||||
args.results_dir, "%s-LDBC-results.json" % vendor_reference)
|
||||
vendors[vendor_reference]['color'] = COLORS[vendor_reference]
|
||||
vendors[vendor_reference]['latencies'] = []
|
||||
vendors[vendor_reference]['query_names'] = []
|
||||
assert len(vendors) == 2, "The graph is tailored for only 2 vendors."
|
||||
|
||||
# Collect the benchmark data.
|
||||
print("LDBC Latency Data")
|
||||
for vendor_reference, vendor_data in vendors.items():
|
||||
print("Vendor: %s" % vendor_reference)
|
||||
with open(vendor_data['results_path']) as results_file:
|
||||
results_data = json.load(results_file)
|
||||
for query_data in results_data["all_metrics"]:
|
||||
mean_runtime = query_data["run_time"]["mean"]
|
||||
query_name = query_data['name']
|
||||
print("%s -> %sms" % (query_name, str(mean_runtime)))
|
||||
vendor_data['latencies'].append(mean_runtime)
|
||||
vendor_data['query_names'].append(query_name)
|
||||
|
||||
# Consistency check.
|
||||
all_query_names = [tuple(vd['query_names']) for vd in vendors.values()]
|
||||
assert len(set(all_query_names)) == 1, \
|
||||
"Queries between different vendors are different!"
|
||||
query_names = all_query_names[0]
|
||||
|
||||
# Plot.
|
||||
ind = np.arange(len(query_names)) # the x locations for the groups
|
||||
width = 0.40 # the width of the bars
|
||||
fig, ax = plt.subplots() # figure setup
|
||||
ax.set_ylabel('Mean Latency (ms)') # YAxis title
|
||||
ax.set_facecolor('#dcdcdc') # plot bg color (light gray)
|
||||
ax.set_xticks(ind + width / len(vendors)) # TODO: adjust (more vendors)
|
||||
# IMPORTANT! Long query names on the XAxis don't look compelling.
|
||||
ax.set_xticklabels(['Q%s' % x for x in range(len(query_names))])
|
||||
# set only horizontal grid lines
|
||||
for line in ax.get_xgridlines():
|
||||
line.set_linestyle(' ')
|
||||
for line in ax.get_ygridlines():
|
||||
line.set_linestyle('--')
|
||||
ax.set_axisbelow(True) # put the grid below all other elements
|
||||
plt.grid(True) # show grid
|
||||
# Draw logo or plot title
|
||||
if args.logo_path is None:
|
||||
ax.set_title(args.plot_title)
|
||||
else:
|
||||
# TODO: improve the logo positioning
|
||||
im = plt.imread(get_sample_data(args.logo_path))
|
||||
plt.gcf().subplots_adjust(top=0.85)
|
||||
newax = fig.add_axes([0.4, 0.75, 0.2, 0.25], anchor='N')
|
||||
newax.imshow(im)
|
||||
newax.axis('off')
|
||||
# Draw bars
|
||||
for index, vendor_data in enumerate(vendors.values()):
|
||||
rects = ax.bar(ind + index * width, vendor_data['latencies'], width,
|
||||
color=vendor_data['color'])
|
||||
vendor_data['rects'] = rects
|
||||
autolabel(ax, rects)
|
||||
rects = [vd['rects'][0] for vd in vendors.values()]
|
||||
titles = [vd['title'] for vd in vendors.values()]
|
||||
ax.legend(rects, titles) # Draw the legend.
|
||||
plt.show()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user