9d1265f41f
Summary: Changed harness output to be compatible with script. Changed cppcheck message formatting. Changed qa message formatting. Changed macro benchmark summary icons. Fixed newlines in status generators. Added group name to harness summary. Macro benchmark summary now handles different files. Added more output options to summary script. Diff builds now build parent for performance comparison. Reviewers: buda, mislav.bradac Reviewed By: mislav.bradac Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D706
133 lines
4.0 KiB
Python
Executable File
133 lines
4.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
"""
|
|
Continuous integration toolkit. The purpose of this script is to generate
|
|
everything which is needed for the CI environment.
|
|
|
|
List of responsibilities:
|
|
* execute default suites
|
|
* terminate execution if any of internal scenarios fails
|
|
* creates the report file that is needed by the Apollo plugin
|
|
to post the status on Phabricator. (.quality_assurance_status)
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import logging
|
|
import subprocess
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
# constants
|
|
memgraph_suite = "memgraph_V1"
|
|
extra_suites = ["openCypher_M06"]
|
|
results_folder = os.path.join("tck_engine", "results")
|
|
suite_suffix = "memgraph-{}.json"
|
|
qa_status_path = ".quality_assurance_status"
|
|
|
|
|
|
def get_newest_path(folder, suffix):
|
|
"""
|
|
:param folder: Scanned folder.
|
|
:param suffix: File suffix.
|
|
|
|
:return: Path to the newest file in the folder with the specified suffix.
|
|
"""
|
|
name_list = sorted(filter(lambda x: x.endswith(suffix),
|
|
os.listdir(folder)))
|
|
if len(name_list) <= 0:
|
|
sys.exit("Unable to find any file with suffix %s in folder %s!" %
|
|
(suffix, folder))
|
|
return os.path.join(folder, name_list.pop())
|
|
|
|
|
|
def generate_status(suite, f, required = False):
|
|
"""
|
|
:param suite: Test suite name.
|
|
:param f: Json file with status report.
|
|
:param required: Adds status ticks to the message if required.
|
|
|
|
:return: Status string.
|
|
"""
|
|
result = json.load(f)
|
|
total = result["total"]
|
|
passed = result["passed"]
|
|
ratio = passed / total
|
|
msg = "{} / {} //({:.2%})//".format(passed, total, ratio)
|
|
if required:
|
|
if passed == total:
|
|
msg += " {icon check color=green}"
|
|
else:
|
|
msg += " {icon times color=red}"
|
|
return (msg, passed, total)
|
|
|
|
|
|
def generate_remarkup(data):
|
|
"""
|
|
:param data: Tabular data to convert to remarkup.
|
|
|
|
:return: Remarkup formatted status string.
|
|
"""
|
|
ret = "==== Quality assurance status: ====\n\n"
|
|
ret += "<table>\n"
|
|
for row in data:
|
|
ret += " <tr>\n"
|
|
for item in row:
|
|
if row == data[0]:
|
|
fmt = " <th>{}</th>\n"
|
|
else:
|
|
fmt = " <td>{}</td>\n"
|
|
ret += fmt.format(item)
|
|
ret += " </tr>\n"
|
|
ret += "</table>\n"
|
|
return ret
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# logger config
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
# run suites
|
|
log.info("Starting Memgraph scenarios.")
|
|
subprocess.run(["./run", "--test-suite", memgraph_suite], check = True)
|
|
for suite in extra_suites:
|
|
log.info("Starting extra suite '{}' scenarios.".format(suite))
|
|
subprocess.run(["./run", "--test-suite", suite])
|
|
|
|
# get data files (memgraph internal test + openCypher TCK test results)
|
|
memgraph_result_path = get_newest_path(results_folder,
|
|
suite_suffix.format(memgraph_suite))
|
|
log.info("Memgraph result path is {}".format(memgraph_result_path))
|
|
|
|
# status table headers
|
|
status_data = [["Suite", "Scenarios"]]
|
|
|
|
# read internal scenarios
|
|
with open(memgraph_result_path) as f:
|
|
memgraph_status, memgraph_passed, memgraph_total \
|
|
= generate_status(memgraph_suite, f, required = True)
|
|
status_data.append([memgraph_suite, memgraph_status])
|
|
|
|
# read extra scenarios
|
|
for suite in extra_suites:
|
|
result_path = get_newest_path(results_folder, suite_suffix.format(suite))
|
|
log.info("Extra suite '{}' result path is {}".format(suite, result_path))
|
|
with open(result_path) as f:
|
|
suite_status, _, _ = generate_status(suite, f)
|
|
status_data.append([suite, suite_status])
|
|
|
|
# create status message
|
|
qa_status_message = generate_remarkup(status_data)
|
|
|
|
# create the report file
|
|
with open(qa_status_path, "w") as f:
|
|
f.write(qa_status_message)
|
|
|
|
log.info("Status is generated in %s" % qa_status_path)
|
|
|
|
if memgraph_total != memgraph_passed:
|
|
sys.exit("There is a problem with internal scenarios! %s"
|
|
% memgraph_status)
|