Added qa tests to apollo generate.

Summary:
The main init script now calls qa init.

QA CI script now runs the tests.

QA run script now returns a good exit code.

Explicitly bind to 127.0.0.1.

localhost on some machines resolves to an IPv6 address, and Memgraph
listens exclusively on IPv4.

Reviewers: buda

Reviewed By: buda

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D583

init

@@ -81,4 +81,9 @@ cd libs
 ./setup.sh
 cd ..
 
+# setup qa dependencies
+cd tests/qa
+./init
+cd ../..
+
 echo "Done installing dependencies for Memgraph"

tests/qa/continuous_integration

@@ -6,6 +6,7 @@ Continuous integration toolkit. The purpose of this script is to generate
 everything which is needed for the CI environment.
 
 List of responsibilities:
+    * execute default suites
     * terminate execution if any of internal scenarios fails
     * creates the report file that is needed by the Jenkins plugin
       to post the status on Phabricator. (.quality_assurance_status)
@@ -15,15 +16,15 @@ import os
 import sys
 import json
 import logging
+import subprocess
 
 log = logging.getLogger(__name__)
 
 # constants
 memgraph_suite = "memgraph_V1"
-opencypher_suite = "openCypher_M06"
+extra_suites = ["openCypher_M06"]
 results_folder = os.path.join("tck_engine", "results")
-memgraph_suffix = "memgraph-%s.json" % memgraph_suite
+suite_suffix = "memgraph-{}.json"
-opencypher_suffix = "memgraph-%s.json" % opencypher_suite
 qa_status_path = ".quality_assurance_status"
 
 
@@ -60,26 +61,37 @@ if __name__ == "__main__":
     # logger config
     logging.basicConfig(level=logging.INFO)
 
+    # run suites
+    log.info("Starting Memgraph scenarios.")
+    subprocess.run(["./run", "--test-suite", memgraph_suite], check = True)
+    for suite in extra_suites:
+        log.info("Starting extra suite '{}' scenarios.".format(suite))
+        subprocess.run(["./run", "--test-suite", suite])
+
     # get data files (memgraph internal test + openCypher TCK test results)
-    memgraph_result_path = get_newest_path(results_folder, memgraph_suffix)
+    memgraph_result_path = get_newest_path(results_folder,
-    log.info("Memgraph result path is %s" % memgraph_result_path)
+            suite_suffix.format(memgraph_suite))
-    opencypher_result_path = get_newest_path(results_folder, opencypher_suffix)
+    log.info("Memgraph result path is {}".format(memgraph_result_path))
-    log.info("openCypher result path is %s" % opencypher_result_path)
 
     # read internal scenarios
     with open(memgraph_result_path) as f:
         memgraph_status, memgraph_passed, memgraph_total \
             = generate_status(memgraph_suite, f)
 
-    # read opencypher scenarios
+    # create status message
-    with open(opencypher_result_path) as f:
+    qa_status_message = "Quality Assurance Status\n" + memgraph_status + "\n"
-        opencypher_status, _, _ = generate_status(opencypher_suite, f)
+
+    # read extra scenarios
+    for suite in extra_suites:
+        result_path = get_newest_path(results_folder, suite_suffix.format(suite))
+        log.info("Extra suite '{}' result path is {}".format(suite, result_path))
+        with open(result_path) as f:
+            suite_status, _, _ = generate_status(suite, f)
+        qa_status_message += suite_status + "\n"
 
     # create the report file
     with open(qa_status_path, "w") as f:
-        f.write("Quality Assurance Status\n")
+        f.write(qa_status_message)
-        f.write("%s\n" % memgraph_status)
-        f.write("%s\n" % opencypher_status)
 
     log.info("Status is generated in %s" % qa_status_path)
 

tests/qa/run

@@ -87,4 +87,4 @@ if [[ $? -ne 0 ]]; then
 fi
 
 python3 tck_engine/test_executor.py $tck_flags
-cleanup_and_exit 0
+cleanup_and_exit $?

tests/qa/tck_engine/test_executor.py

@@ -18,8 +18,8 @@ def parse_args():
     argp.add_argument("--db-user", default="neo4j", help="Default is neo4j.")
     argp.add_argument(
         "--db-pass", default="1234", help="Default is 1234.")
-    argp.add_argument("--db-uri", default="bolt://localhost:7687",
+    argp.add_argument("--db-uri", default="bolt://127.0.0.1:7687",
-                      help="Default is bolt://localhost:7687.")
+                      help="Default is bolt://127.0.0.1:7687.")
     argp.add_argument("--output-folder", default="tck_engine/results/",
                       help="Test result output folder, default is results/.")
     argp.add_argument("--logging", default="DEBUG", choices=[

tools/apollo_generate

@@ -71,8 +71,12 @@ if os.path.exists(OUTPUT_DIR):
 os.makedirs(OUTPUT_DIR)
 
 # store memgraph binary to archive
-binary_name = run_cmd(["find", ".", "-maxdepth", "1", "-executable", "-name", "memgraph*"], BUILD_DIR).split("\n")[0][2:]
+binary_name = run_cmd(["find", ".", "-maxdepth", "1", "-executable", "-type",
+        "f", "-name", "memgraph*"], BUILD_DIR).split("\n")[0][2:]
+binary_link_name = run_cmd(["find", ".", "-maxdepth", "1", "-executable", "-type",
+        "l", "-name", "memgraph*"], BUILD_DIR).split("\n")[0][2:]
 binary_path = os.path.join(BUILD_DIR, binary_name)
+binary_link_path = os.path.join(BUILD_DIR, binary_link_name)
 config_path = os.path.join(BASE_DIR, "config")
 config_copy_path = os.path.join(BUILD_DIR, "config")
 if os.path.exists(config_copy_path):
@@ -147,6 +151,17 @@ for test in tests:
 
     RUNS.append(run)
 
+# quality assurance tests
+qa_path = os.path.join(BASE_DIR, "tests", "qa")
+infile = create_archive("quality_assurance", [qa_path, binary_path,
+        binary_link_path, config_path], cwd = WORKSPACE_DIR)
+commands = "cd {}/tests/qa\n./continuous_integration\n".format(
+        BASE_DIR_NAME)
+RUNS.append(generate_run("quality_assurance", commands = commands,
+        infile = infile, outfile_paths = "\./{}/tests/qa/"
+        "\.quality_assurance_status".format(
+        BASE_DIR_NAME)))
+
 # store ARCHIVES and RUNS
 store_metadata(OUTPUT_DIR, "archives", ARCHIVES)
 store_metadata(OUTPUT_DIR, "runs", RUNS)