Begin qa refactor

Reviewers: mferencevic, matej.gradicek, buda

Reviewed By: buda

Differential Revision: https://phabricator.memgraph.io/D225

README.md

@@ -20,15 +20,15 @@ Python script used to run tck tests against memgraph. To run script execute:
 Script uses Behave to run Cucumber tests.
 
 The following tck tests have been changed:
-	
+
-    1. Tests where example injection did not work. Behave stores the first row 
+    1. Tests where example injection did not work. Behave stores the first row
        in Cucumber tables as headings and the example injection is not working in
        headings. To correct this behavior, one row was added to tables where
        injection was used.
 
     2. Tests where the results were not always in the same order. Query does not
-       specify the result order, but tests specified it. It led to the test failure. 
+       specify the result order, but tests specified it. It led to the test failure.
-       To correct tests, tag "the result should be" was changed with a 
+       To correct tests, tag "the result should be" was changed with a
        tag "the result should be (ignoring element order for lists)".
 
 Comparability.feature tests are failing because integers are compared to strings
@@ -45,13 +45,13 @@ TCK Engine problems:
 
     2. Properties side effects
          | +properties | 1 |
-         | -properties | 1 | 
+         | -properties | 1 |
 
          Database is returning properties_set, not properties_created and properties_deleted.
- 
+
 ## KPI Service
 
-Flask application used to get results from executing tests with TCK Engine. 
+Flask application used to get results from executing tests with TCK Engine.
 Application can be ran executing:
-	
+
     1. python3 kpi_service/kpi_service.py

init

@@ -1,20 +1,31 @@
 #!/bin/bash
 
+function print_usage_and_exit {
+    echo "./init [--no-clone-dependencies] [--arc-diff DIFF_ID]"
+    exit 1
+}
+
 # read arguments
+arcanist_diff_id=""
 clone_dependencies=true
-while [[ $# -gt 0 ]]
+while [[ $# -gt 0 ]]; do
-do
+    case $1 in
-key="$1"
+        --no-clone-dependencies)
-case $key in
+            clone_dependencies=false
-    --no-clone-dependencies)
+            shift
-    clone_dependencies=false
+            ;;
-    shift # past argument
+        --arc-diff)
-    ;;
+            if [[ -z $2 ]]; then
-    *)
+                print_usage_and_exit
-	# unknown option
+            fi
-    ;;
+            arcanist_diff_id=$2
-esac
+            shift 2
-shift # past argument
+            ;;
+        *)
+            # unknown option
+            print_usage_and_exit
+            ;;
+    esac
 done
 
 # exit if any subcommand returns a non-zero status
@@ -22,26 +33,23 @@ set -e
 
 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-# optional argument; if diff_id exists arcanist patch will be applied
-arcanist_diff_id=${1:-""}
-
 # install all dependencies on debian based operating systems
-for pkg in git arcanist python3-virtualenv python3-pip; do
+for pkg in git arcanist python-virtualenv python3-pip; do
     dpkg -s $pkg 2>/dev/null >/dev/null || sudo apt-get -y install $pkg
 done
 
 # TODO: use pullbot and read a password from program arg or env var
 #       (in order to use this script inside CI infrastructure)
 
-# clean tests folder
+# create tests folder
 tests_folder=${script_dir}/tests
 mkdir -p ${tests_folder}
 
 # clone dressipi's tests
 cd ${tests_folder}
-if ${clone_dependencies} ; then
+if ${clone_dependencies}; then
-rm -rf ${tests_folder}/*
+    rm -rf ${tests_folder}/*
-git clone https://phabricator.memgraph.io/source/pilot_dressipi.git
+    git clone https://phabricator.memgraph.io/source/pilot_dressipi.git
 fi
 
 # clean dbms folder
@@ -51,8 +59,8 @@ mkdir -p ${dbms_folder}
 # clone memgraph & checkout right commit
 cd ${dbms_folder}
 if ${clone_dependencies} ; then
-rm -rf ${dbms_folder}/*
+    rm -rf ${dbms_folder}/*
-git clone https://phabricator.memgraph.io/diffusion/MG/memgraph.git
+    git clone https://phabricator.memgraph.io/diffusion/MG/memgraph.git
 fi
 memgraph_folder=${dbms_folder}/memgraph
 cd ${memgraph_folder}
@@ -63,6 +71,13 @@ if [[ !  -z  ${arcanist_diff_id}  ]]; then
     # and it fails if user isn't set (inside CI infrastructure the user
     # probably won't be defined because CI infrastructure doesn't push
     # any code back to the repository)
-    arc parch ${arcanist_diff_id} --nocommit
+    arc patch ${arcanist_diff_id} --nocommit
 fi
 ./init
+
+# compile memgraph
+memgraph_build_dir=${script_dir}/dbms/memgraph/build
+cd ${memgraph_build_dir}
+cmake ..
+make -j8
+make copy_hardcoded_queries

requirements.txt

@@ -5,7 +5,7 @@ Flask==0.12
 itsdangerous==0.24
 Jinja2==2.9.5
 MarkupSafe==1.0
-neo4j-driver==1.0.2
+neo4j-driver==1.2.0
 packaging==16.8
 parse==1.8.0
 parse-type==0.3.4

run

@@ -1,22 +1,5 @@
 #!/bin/bash
 
-# read arguments
-memgraph_compile=true
-while [[ $# -gt 0 ]]
-do
-key="$1"
-case $key in
-    --no-memgraph-compile)
-    memgraph_compile=false
-    shift # past argument
-    ;;
-    *)
-	# unknown option
-    ;;
-esac
-shift # past argument
-done
-
 # exit if any subcommand returns a non-zero status
 set -e
 
@@ -27,14 +10,6 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 memgraph_src_dir=${script_dir}/dbms/memgraph
 memgraph_build_dir=${script_dir}/dbms/memgraph/build
 
-# compile memgraph
-cd ${memgraph_build_dir}
-if ${memgraph_compile} ; then
-cmake ..
-make -j8
-make copy_hardcoded_queries
-fi
-
 # setup dressipi
 cd ${script_dir}
 # setup ve
@@ -48,17 +23,18 @@ pip3 install -r requirements.txt
 cd ${memgraph_build_dir}
 # binary is available after the build
 binary_name=$(find ${memgraph_build_dir}/ -maxdepth 1 -executable -name "memgraph*")
-# get full path to memgraph config
+
-config_path="${memgraph_src_dir}/config/memgraph.yaml"
+# get full path to memgraph config for hardcoded queries
+hardcoded_queries_config_path="${memgraph_src_dir}/config/hardcoded_queries_memgraph.yaml"
 # run memgraph
-MEMGRAPH_CONFIG=${config_path} ${binary_name} &
+MEMGRAPH_CONFIG=${hardcoded_queries_config_path} ${binary_name} &
 
 function cleanup_and_exit {
     pkill -9 -f "${binary_name}"
     exit $1
 }
 
-# the script has to be carefull because one process has been detached
+# the script has to be careful because one process has been detached
 set +e
 
 ## run tests
@@ -71,7 +47,18 @@ if [[ ${exit_code} != 0 ]]; then
     cleanup_and_exit ${exit_code}
 fi
 
-# run TCK test
+# kill memgraph for hardcoded queries
+pkill -9 -f "${binary_name}"
+# get full path to memgraph config for interpreted queries
+config_path="${memgraph_src_dir}/config/memgraph.yaml"
+# run memgraph
+MEMGRAPH_CONFIG=${config_path} ${binary_name} &
+
+# run custom scenarios
 cd ${script_dir}
-python tck_engine/test_executor.py --root tck_engine/tests/openCypher_M05/tck/features --no-side-effects --db memgraph
+python tck_engine/test_executor.py \
+    --root tck_engine/tests/memgraph_V1/features \
+    --graphs-root tck_engine/tests/memgraph_V1/graphs \
+    --no-side-effects \
+    --db memgraph
 cleanup_and_exit ${exit_code}

tck_engine/environment.py

@@ -14,21 +14,19 @@ def before_scenario(context, step):
     context.test_parameters = TestParameters()
     context.graph_properties = GraphProperties()
     context.exception = None
-    context.driver = create_db_driver(context)
-    context.session = context.driver.session()
 
 
 def after_scenario(context, scenario):
     test_results.add_test(scenario.status)
-    context.session.close()
-
-
 
 def before_all(context):
+    context.driver = create_db_driver(context)
     set_logging(context)
 
 
 def after_all(context):
+    context.driver.close()
+
     ts = time.time()
     timestamp = datetime.datetime.fromtimestamp(ts).strftime("%Y_%m_%d__%H_%M")
 

tck_engine/steps/database.py

@@ -13,16 +13,20 @@ def query(q, context, params={}):
     """
     results_list = []
 
-    if context.config.database == "neo4j":
+    if (context.config.database == "neo4j" or
-        session = context.session
+        context.config.database == "memgraph"):
+        session = context.driver.session()
         try:
             # executing query
             results = session.run(q, params)
-            summary = results.summary()
             if not context.config.no_side_effects:
+                summary = results.summary()
                 add_side_effects(context, summary.counters)
             results_list = list(results)
             """
+            This code snippet should replace code which is now
+            executing queries when session.transactions will be supported.
+
             with session.begin_transaction() as tx:
                 results = tx.run(q, params)
                 summary = results.summary()
@@ -30,16 +34,13 @@ def query(q, context, params={}):
                     add_side_effects(context, summary.counters)
                 results_list = list(results)
                 tx.success = True
-
-            This code snippet should replace code which is now
-            executing queries when session.transactions will be supported.
             """
         except Exception as e:
             # exception
             context.exception = e
             context.log.info('%s', str(e))
-            # not working if removed
+        finally:
-            query("match (n) detach delete(n)", context)
+            session.close()
     return results_list
 
 

tck_engine/steps/graph.py

@@ -3,15 +3,22 @@ import os
 from behave import *
 
 
+def clear_graph(context):
+    database.query("MATCH (n) DETACH DELETE n", context)
+    if context.exception != None:
+        context.exception = None
+        database.query("MATCH (n) DETACH DELETE n", context)
+
+
 @given('an empty graph')
 def empty_graph_step(context):
-    database.query("MATCH (n) DETACH DELETE n", context)
+    clear_graph(context)
     context.graph_properties.set_beginning_parameters()
 
 
 @given('any graph')
 def any_graph_step(context):
-    database.query("MATCH (n) DETACH DELETE n", context)
+    clear_graph(context)
     context.graph_properties.set_beginning_parameters()
 
 
@@ -27,7 +34,7 @@ def create_graph(name, context):
     executes queries written in a .cypher file separated by ';'
     and sets graph properties to beginning values.
     """
-    database.query("MATCH (n) DETACH DELETE n", context)
+    clear_graph(context)
     path = find_graph_path(name, context.config.graphs_root)
 
     q_marks = ["'", '"', '`']

tck_engine/test_executor.py

@@ -19,7 +19,7 @@ def parse_args():
                       "neo4j", "memgraph"], help="Default is neo4j.")
     argp.add_argument("--db-user", default="neo4j", help="Default is neo4j.")
     argp.add_argument(
-        "--db-pass", default="memgraph", help="Default is memgraph.")
+        "--db-pass", default="1234", help="Default is 1234.")
     argp.add_argument("--db-uri", default="bolt://localhost:7687",
                       help="Default is bolt://localhost:7687.")
     argp.add_argument("--output-folder", default="tck_engine/results/",
@@ -29,6 +29,12 @@ def parse_args():
     return argp.parse_args()
 
 
+def add_config(option, dictionary):
+    configuration.options.append(
+        ((option,), dictionary)
+    )
+
+
 def main():
     """
     Script used to run behave tests with given options. List of
@@ -77,10 +83,5 @@ def main():
     behave_main(behave_options)
 
 
-def add_config(option, dictionary):
-    configuration.options.append(
-        ((option,), dictionary)
-    )
-
 if __name__ == '__main__':
     main()

tck_engine/tests/memgraph_V1/features/test01.feature

@@ -6,7 +6,7 @@ Feature: Test01
             CREATE (n)
             """
         Then the result should be empty
-    
+
     Scenario: Create node with label without clearing database
         When executing query:
             """
@@ -20,7 +20,7 @@ Feature: Test01
             CREATE (n{a: 1})
             """
         Then the result should be empty
-    
+
     Scenario: Create node with float property without clearing database
         When executing query:
             """
@@ -48,21 +48,21 @@ Feature: Test01
             CREATE (n{a: NULL})
             """
         Then the result should be empty
-    
+
     Scenario: Create node with properties without clearing database
         When executing query:
             """
             CREATE (n{a: 1.0, b: false, c: 1, d: 'neki"string"', e: NULL})
             """
         Then the result should be empty
-    
+
     Scenario: Create node with properties without clearing database
         When executing query:
             """
             CREATE (n:L:K:T {a: 1.0, b: false, c: 1, d: 'neki"string"', e: NULL})
             """
         Then the result should be empty
-    
+
     Scenario: Create multiple nodes connected by relationships
         When executing query:
             """
@@ -85,7 +85,7 @@ Feature: Test01
         Then the result should be empty
 
 
-    
+
 
     Scenario: Create empty node without clearing database:
         When executing query:
@@ -95,7 +95,7 @@ Feature: Test01
         Then the result should be:
             | n |
             |( )|
-    
+
     Scenario: Create node with labels without clearing database and return it
         When executing query:
             """
@@ -433,4 +433,3 @@ Feature: Test01
             |              n                |
             |     (:Person {age: 20})       |
             |  (:Person :Student {age: 20}) |
-