diff --git a/tests/e2e/high_availability_experimental/single_coordinator.py b/tests/e2e/high_availability_experimental/single_coordinator.py index d490a36ba..214fa5905 100644 --- a/tests/e2e/high_availability_experimental/single_coordinator.py +++ b/tests/e2e/high_availability_experimental/single_coordinator.py @@ -147,6 +147,73 @@ def test_replication_works_on_failover(): interactive_mg_runner.stop_all(MEMGRAPH_INSTANCES_DESCRIPTION) +def test_replication_works_on_instance_down(): + # Goal of this test is to check the replication works after failover command. + # 1. We start all replicas, main and coordinator manually: we want to be able to kill them ourselves without relying on external tooling to kill processes. + # 2. We check that main has correct state + # 3. We kill main + # 4. We check that coordinator and new main have correct state + # 5. We insert one vertex on new main + # 6. We check that vertex appears on new replica + safe_execute(shutil.rmtree, TEMP_DIR) + + # 1 + interactive_mg_runner.start_all(MEMGRAPH_INSTANCES_DESCRIPTION) + + # 2 + main_cursor = connect(host="localhost", port=7687).cursor() + expected_data_on_main = [ + ("instance_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("instance_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ] + actual_data_on_main = sorted(list(execute_and_fetch_all(main_cursor, "SHOW REPLICAS;"))) + assert actual_data_on_main == expected_data_on_main + + # 3 + interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_3") + + # 4 + coord_cursor = connect(host="localhost", port=7690).cursor() + + def retrieve_data_show_repl_cluster(): + return sorted(list(execute_and_fetch_all(coord_cursor, "SHOW INSTANCES;"))) + + expected_data_on_coord = [ + ("coordinator_1", "127.0.0.1:10111", "", True, "coordinator"), + ("instance_1", "", "127.0.0.1:10011", True, "main"), + ("instance_2", "", "127.0.0.1:10012", True, "replica"), + ("instance_3", "", "127.0.0.1:10013", False, "unknown"), + ] + mg_sleep_and_assert(expected_data_on_coord, retrieve_data_show_repl_cluster) + + new_main_cursor = connect(host="localhost", port=7688).cursor() + + def retrieve_data_show_replicas(): + return sorted(list(execute_and_fetch_all(new_main_cursor, "SHOW REPLICAS;"))) + + expected_data_on_new_main = [ + ("instance_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ("instance_3", "127.0.0.1:10003", "sync", 0, 0, "invalid"), + ] + mg_sleep_and_assert(expected_data_on_new_main, retrieve_data_show_replicas) + + interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_3") + expected_data_on_new_main = [ + ("instance_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ("instance_3", "127.0.0.1:10003", "sync", 0, 0, "ready"), + ] + mg_sleep_and_assert(expected_data_on_new_main, retrieve_data_show_replicas) + + # 5 + execute_and_fetch_all(new_main_cursor, "CREATE ();") + + # 6 + alive_replica_cursror = connect(host="localhost", port=7689).cursor() + res = execute_and_fetch_all(alive_replica_cursror, "MATCH (n) RETURN count(n) as count;")[0][0] + assert res == 1, "Vertex should be replicated" + interactive_mg_runner.stop_all(MEMGRAPH_INSTANCES_DESCRIPTION) + + def test_show_instances(): safe_execute(shutil.rmtree, TEMP_DIR) interactive_mg_runner.start_all(MEMGRAPH_INSTANCES_DESCRIPTION)