diff --git a/init b/init index e06e1921d..5888ad228 100755 --- a/init +++ b/init @@ -81,4 +81,9 @@ cd libs ./setup.sh cd .. +# setup qa dependencies +cd tests/qa +./init +cd ../.. + echo "Done installing dependencies for Memgraph" diff --git a/tests/qa/continuous_integration b/tests/qa/continuous_integration index f7d225127..37d3dc2c3 100755 --- a/tests/qa/continuous_integration +++ b/tests/qa/continuous_integration @@ -6,6 +6,7 @@ Continuous integration toolkit. The purpose of this script is to generate everything which is needed for the CI environment. List of responsibilities: + * execute default suites * terminate execution if any of internal scenarios fails * creates the report file that is needed by the Jenkins plugin to post the status on Phabricator. (.quality_assurance_status) @@ -15,15 +16,15 @@ import os import sys import json import logging +import subprocess log = logging.getLogger(__name__) # constants memgraph_suite = "memgraph_V1" -opencypher_suite = "openCypher_M06" +extra_suites = ["openCypher_M06"] results_folder = os.path.join("tck_engine", "results") -memgraph_suffix = "memgraph-%s.json" % memgraph_suite -opencypher_suffix = "memgraph-%s.json" % opencypher_suite +suite_suffix = "memgraph-{}.json" qa_status_path = ".quality_assurance_status" @@ -60,26 +61,37 @@ if __name__ == "__main__": # logger config logging.basicConfig(level=logging.INFO) + # run suites + log.info("Starting Memgraph scenarios.") + subprocess.run(["./run", "--test-suite", memgraph_suite], check = True) + for suite in extra_suites: + log.info("Starting extra suite '{}' scenarios.".format(suite)) + subprocess.run(["./run", "--test-suite", suite]) + # get data files (memgraph internal test + openCypher TCK test results) - memgraph_result_path = get_newest_path(results_folder, memgraph_suffix) - log.info("Memgraph result path is %s" % memgraph_result_path) - opencypher_result_path = get_newest_path(results_folder, opencypher_suffix) - log.info("openCypher result path is %s" % opencypher_result_path) + memgraph_result_path = get_newest_path(results_folder, + suite_suffix.format(memgraph_suite)) + log.info("Memgraph result path is {}".format(memgraph_result_path)) # read internal scenarios with open(memgraph_result_path) as f: memgraph_status, memgraph_passed, memgraph_total \ = generate_status(memgraph_suite, f) - # read opencypher scenarios - with open(opencypher_result_path) as f: - opencypher_status, _, _ = generate_status(opencypher_suite, f) + # create status message + qa_status_message = "Quality Assurance Status\n" + memgraph_status + "\n" + + # read extra scenarios + for suite in extra_suites: + result_path = get_newest_path(results_folder, suite_suffix.format(suite)) + log.info("Extra suite '{}' result path is {}".format(suite, result_path)) + with open(result_path) as f: + suite_status, _, _ = generate_status(suite, f) + qa_status_message += suite_status + "\n" # create the report file with open(qa_status_path, "w") as f: - f.write("Quality Assurance Status\n") - f.write("%s\n" % memgraph_status) - f.write("%s\n" % opencypher_status) + f.write(qa_status_message) log.info("Status is generated in %s" % qa_status_path) diff --git a/tests/qa/run b/tests/qa/run index a92e68c95..a8fefa451 100755 --- a/tests/qa/run +++ b/tests/qa/run @@ -87,4 +87,4 @@ if [[ $? -ne 0 ]]; then fi python3 tck_engine/test_executor.py $tck_flags -cleanup_and_exit 0 +cleanup_and_exit $? diff --git a/tests/qa/tck_engine/test_executor.py b/tests/qa/tck_engine/test_executor.py index dac8d56b7..039e8459d 100644 --- a/tests/qa/tck_engine/test_executor.py +++ b/tests/qa/tck_engine/test_executor.py @@ -18,8 +18,8 @@ def parse_args(): argp.add_argument("--db-user", default="neo4j", help="Default is neo4j.") argp.add_argument( "--db-pass", default="1234", help="Default is 1234.") - argp.add_argument("--db-uri", default="bolt://localhost:7687", - help="Default is bolt://localhost:7687.") + argp.add_argument("--db-uri", default="bolt://127.0.0.1:7687", + help="Default is bolt://127.0.0.1:7687.") argp.add_argument("--output-folder", default="tck_engine/results/", help="Test result output folder, default is results/.") argp.add_argument("--logging", default="DEBUG", choices=[ diff --git a/tools/apollo_generate b/tools/apollo_generate index 9093689b7..176eed64b 100755 --- a/tools/apollo_generate +++ b/tools/apollo_generate @@ -71,8 +71,12 @@ if os.path.exists(OUTPUT_DIR): os.makedirs(OUTPUT_DIR) # store memgraph binary to archive -binary_name = run_cmd(["find", ".", "-maxdepth", "1", "-executable", "-name", "memgraph*"], BUILD_DIR).split("\n")[0][2:] +binary_name = run_cmd(["find", ".", "-maxdepth", "1", "-executable", "-type", + "f", "-name", "memgraph*"], BUILD_DIR).split("\n")[0][2:] +binary_link_name = run_cmd(["find", ".", "-maxdepth", "1", "-executable", "-type", + "l", "-name", "memgraph*"], BUILD_DIR).split("\n")[0][2:] binary_path = os.path.join(BUILD_DIR, binary_name) +binary_link_path = os.path.join(BUILD_DIR, binary_link_name) config_path = os.path.join(BASE_DIR, "config") config_copy_path = os.path.join(BUILD_DIR, "config") if os.path.exists(config_copy_path): @@ -147,6 +151,17 @@ for test in tests: RUNS.append(run) +# quality assurance tests +qa_path = os.path.join(BASE_DIR, "tests", "qa") +infile = create_archive("quality_assurance", [qa_path, binary_path, + binary_link_path, config_path], cwd = WORKSPACE_DIR) +commands = "cd {}/tests/qa\n./continuous_integration\n".format( + BASE_DIR_NAME) +RUNS.append(generate_run("quality_assurance", commands = commands, + infile = infile, outfile_paths = "\./{}/tests/qa/" + "\.quality_assurance_status".format( + BASE_DIR_NAME))) + # store ARCHIVES and RUNS store_metadata(OUTPUT_DIR, "archives", ARCHIVES) store_metadata(OUTPUT_DIR, "runs", RUNS)