Improve e2e Kafka and Pulsar testing (#1604)

.github/workflows/diff.yaml

@@ -229,10 +229,6 @@ jobs:
           # branches and tags. (default: 1)
           fetch-depth: 0
 
-      - name: Check e2e service dependencies
-        run: |
-          cd tests/e2e
-          ./dependency_check.sh
 
       - name: Build release binaries
         run: |
@@ -271,6 +267,13 @@ jobs:
           cd build
           ctest -R memgraph__unit --output-on-failure -j$THREADS
 
+      - name: Ensure Kafka and Pulsar are up
+        run: |
+          cd tests/e2e/streams/kafka
+          docker-compose up -d
+          cd ../pulsar
+          docker-compose up -d
+
       - name: Run e2e tests
         run: |
           cd tests
@@ -279,6 +282,14 @@ jobs:
           cd e2e
           ./run.sh
 
+      - name: Ensure Kafka and Pulsar are down
+        if: always()
+        run: |
+          cd tests/e2e/streams/kafka
+          docker-compose down
+          cd ../pulsar
+          docker-compose down
+
       - name: Run stress test (plain)
         run: |
           cd tests/stress

tests/e2e/streams/kafka/docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3'
+services:
+  zookeeper:
+    image: 'bitnami/zookeeper:3.9.1'
+    ports:
+      - '2181:2181'
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+  kafka:
+    image: 'bitnami/kafka:3.6.1'
+    ports:
+      - '9092:9092'
+      - '29092:29092'
+    environment:
+      - KAFKA_BROKER_ID=1
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,PLAINTEXT_HOST://:29092
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092,PLAINTEXT_HOST://localhost:29092
+      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
+      - ALLOW_PLAINTEXT_LISTENER=yes
+    depends_on:
+      - zookeeper

tests/e2e/streams/pulsar/docker-compose.yml

@@ -0,0 +1,9 @@
+version: "3"
+
+services:
+  pulsar:
+    image: 'apachepulsar/pulsar:2.8.1'
+    ports:
+      - '6652:8080'
+      - '6650:6650'
+    entrypoint: [ 'bin/pulsar', 'standalone' ]