29a7b12e22
Summary: The main init script now calls qa init. QA CI script now runs the tests. QA run script now returns a good exit code. Explicitly bind to 127.0.0.1. localhost on some machines resolves to an IPv6 address, and Memgraph listens exclusively on IPv4. Reviewers: buda Reviewed By: buda Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D583
101 lines
3.2 KiB
Python
Executable File
101 lines
3.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
"""
|
|
Continuous integration toolkit. The purpose of this script is to generate
|
|
everything which is needed for the CI environment.
|
|
|
|
List of responsibilities:
|
|
* execute default suites
|
|
* terminate execution if any of internal scenarios fails
|
|
* creates the report file that is needed by the Jenkins plugin
|
|
to post the status on Phabricator. (.quality_assurance_status)
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import logging
|
|
import subprocess
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
# constants
|
|
memgraph_suite = "memgraph_V1"
|
|
extra_suites = ["openCypher_M06"]
|
|
results_folder = os.path.join("tck_engine", "results")
|
|
suite_suffix = "memgraph-{}.json"
|
|
qa_status_path = ".quality_assurance_status"
|
|
|
|
|
|
def get_newest_path(folder, suffix):
|
|
"""
|
|
:param folder: Scanned folder.
|
|
:param suffix: File suffix.
|
|
|
|
:return: Path to the newest file in the folder with the specified suffix.
|
|
"""
|
|
name_list = sorted(filter(lambda x: x.endswith(suffix),
|
|
os.listdir(folder)))
|
|
if len(name_list) <= 0:
|
|
sys.exit("Unable to find any file with suffix %s in folder %s!" %
|
|
(suffix, folder))
|
|
return os.path.join(folder, name_list.pop())
|
|
|
|
|
|
def generate_status(suite, f):
|
|
"""
|
|
:param suite: Test suite name.
|
|
:param f: Json file with status report.
|
|
|
|
:return: Status string.
|
|
"""
|
|
result = json.load(f)
|
|
total = result["total"]
|
|
passed = result["passed"]
|
|
return ("SUITE: %s, PASSED SCENARIOS: %s, TOTAL SCENARIOS: %s (%.2f%%)" %
|
|
(suite, passed, total, 100.0 * passed / total)), passed, total
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# logger config
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
# run suites
|
|
log.info("Starting Memgraph scenarios.")
|
|
subprocess.run(["./run", "--test-suite", memgraph_suite], check = True)
|
|
for suite in extra_suites:
|
|
log.info("Starting extra suite '{}' scenarios.".format(suite))
|
|
subprocess.run(["./run", "--test-suite", suite])
|
|
|
|
# get data files (memgraph internal test + openCypher TCK test results)
|
|
memgraph_result_path = get_newest_path(results_folder,
|
|
suite_suffix.format(memgraph_suite))
|
|
log.info("Memgraph result path is {}".format(memgraph_result_path))
|
|
|
|
# read internal scenarios
|
|
with open(memgraph_result_path) as f:
|
|
memgraph_status, memgraph_passed, memgraph_total \
|
|
= generate_status(memgraph_suite, f)
|
|
|
|
# create status message
|
|
qa_status_message = "Quality Assurance Status\n" + memgraph_status + "\n"
|
|
|
|
# read extra scenarios
|
|
for suite in extra_suites:
|
|
result_path = get_newest_path(results_folder, suite_suffix.format(suite))
|
|
log.info("Extra suite '{}' result path is {}".format(suite, result_path))
|
|
with open(result_path) as f:
|
|
suite_status, _, _ = generate_status(suite, f)
|
|
qa_status_message += suite_status + "\n"
|
|
|
|
# create the report file
|
|
with open(qa_status_path, "w") as f:
|
|
f.write(qa_status_message)
|
|
|
|
log.info("Status is generated in %s" % qa_status_path)
|
|
|
|
if memgraph_total != memgraph_passed:
|
|
sys.exit("There is a problem with internal scenarios! %s"
|
|
% memgraph_status)
|