Add stress test large and improve workflows

.github/workflows/release_build_test.yaml

@@ -13,6 +13,27 @@ on:
         options:
           - Release
           - RelWithDebInfo
+      os:
+        type: choice
+        description: "Target OS on which the release tests will be run."
+        default: 'debian-11'
+        options:
+          - amzn-2
+          - centos-7
+          - centos-9
+          - debian-11
+          - debian-12
+          - fedora-38
+          - fedora-39
+          - rocky-9.3
+          - ubuntu-20.04
+          - ubuntu-22.04
+      toolchain:
+        type: choice
+        description: "Toolchain version with which memgraph will be packaged."
+        default: 'v5'
+        options:
+          - v5
 
   push:
     tags:
@@ -23,11 +44,27 @@ on:
     - cron: "0 22 * * *"
 
 jobs:
-  Test:
+  TestIndividual:
+    if: ${{ !github.event.inputs.os || github.event.inputs.os != 'all' }}
     uses: ./.github/workflows/reusable_release_tests.yaml
     with:
-      os: "debian-11"
-      toolchain: "v5"
+      os: ${{ github.event.inputs.os || 'debian-11' }}
+      toolchain: ${{ github.event.inputs.toolchain || 'v5' }}
+      arch: "amd"
+      runner_arch_label: "X64"
+      threads: 24
+      build_type: ${{ github.event.inputs.build_type || 'Release' }}
+    secrets: inherit
+
+  TestAll:
+    if: ${{ github.event.inputs.os == 'all' }}
+    strategy:
+      matrix:
+        os: [amzn-2, centos-7, centos-9, debian-11, debian-12, fedora-38, fedora-39, rocky-9.3, ubuntu-20.04, ubuntu-22.04]
+    uses: ./.github/workflows/reusable_release_tests.yaml
+    with:
+      os: ${{ matrix.os }}
+      toolchain: ${{ github.event.inputs.toolchain || 'v5' }}
       arch: "amd"
       runner_arch_label: "X64"
       threads: 24
@@ -36,10 +73,11 @@ jobs:
 
   Package:
     if: github.ref_type == 'tag'
+    needs: [TestIndividual]
     uses: ./.github/workflows/reusable_package.yaml
     with:
-      os: "debian-11"
-      toolchain: "v5"
+      os: ${{ github.event.inputs.os || 'debian-11' }}
+      toolchain: ${{ github.event.inputs.toolchain || 'v5' }}
       arch: "amd"
       runner_arch_label: "X64"
       build_type: ${{ github.event.inputs.build_type || 'Release' }}

.github/workflows/reusable_release_tests.yaml

@@ -27,6 +27,14 @@ on:
         type: number
         description: "Number of threads to use for Memgraph build. Default value is 24."
         default: 24
+      run_release_tests:
+        type: string
+        description: "Run all release tests. Default value is true."
+        default: 'true'
+      run_stress_large:
+        type: string
+        description: "Run large stress test. Default value is false."
+        default: 'false'
 
 env:
   OS: ${{ inputs.os }}
@@ -39,6 +47,7 @@ env:
 
 jobs:
   community_build:
+    if: ${{ inputs.run_release_tests == 'true' }}
     name: "Community build"
     runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"]
     timeout-minutes: 60
@@ -94,6 +103,7 @@ jobs:
           stop --remove
 
   coverage_build:
+    if: ${{ inputs.run_release_tests == 'true' }}
     name: "Coverage build"
     runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"]
     timeout-minutes: 60
@@ -166,6 +176,7 @@ jobs:
           stop --remove
 
   debug_build:
+    if: ${{ inputs.run_release_tests == 'true' }}
     name: "Debug build"
     runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"]
     timeout-minutes: 60
@@ -262,6 +273,7 @@ jobs:
           stop --remove
 
   debug_integration_test:
+    if: ${{ inputs.run_release_tests == 'true' }}
     name: "Debug integration tests"
     runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"]
     timeout-minutes: 60
@@ -319,6 +331,7 @@ jobs:
           stop --remove
 
   release_build:
+    if: ${{ inputs.run_release_tests == 'true' }}
     name: "Release build"
     runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"]
     timeout-minutes: 60
@@ -414,6 +427,7 @@ jobs:
           stop --remove
 
   release_benchmark_tests:
+    if: ${{ inputs.run_release_tests == 'true' }}
     name: "Release Benchmark Tests"
     runs-on: [self-hosted, DockerMgBuild, Gen7, "${{ inputs.runner_arch_label }}"]
     timeout-minutes: 60
@@ -488,6 +502,7 @@ jobs:
           stop --remove
 
   release_e2e_test:
+    if: ${{ inputs.run_release_tests == 'true' }}
     name: "Release End-to-end Test"
     runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"]
     timeout-minutes: 60
@@ -561,6 +576,7 @@ jobs:
           stop --remove
 
   release_durability_stress_tests:
+    if: ${{ inputs.run_release_tests == 'true' }}
     name: "Release durability and stress tests"
     runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"]
     timeout-minutes: 60
@@ -645,7 +661,7 @@ jobs:
           stop --remove
 
   release_jepsen_test:
-    if: ${{ inputs.os == 'debian-12' }}
+    if: ${{ inputs.run_release_tests == 'true' && inputs.os == 'debian-12' }}
     name: "Release Jepsen Test"
     runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"]
     timeout-minutes: 60
@@ -713,3 +729,58 @@ jobs:
           --os $OS \
           --arch $ARCH \
           stop --remove
+
+  stress_test_large:
+    if: ${{ inputs.run_stress_large == 'true' }}
+    name: "Stress test large"
+    runs-on: [self-hosted, DockerMgBuild, X64, BigMemory]
+    timeout-minutes: 720
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Spin up mgbuild container
+        run: |
+          ./release/package/mgbuild.sh \
+          --toolchain $TOOLCHAIN \
+          --os $OS \
+          --arch $ARCH \
+          run
+
+      - name: Build release binaries
+        run: |
+          ./release/package/mgbuild.sh \
+          --toolchain $TOOLCHAIN \
+          --os $OS \
+          --arch $ARCH \
+          --build-type $BUILD_TYPE \
+          --threads $THREADS \
+          build-memgraph
+
+      - name: Run stress test (large)
+        run: |
+          ./release/package/mgbuild.sh \
+          --toolchain $TOOLCHAIN \
+          --os $OS \
+          --arch $ARCH \
+          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
+          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
+          test-memgraph stress-large
+
+      - name: Stop mgbuild container
+        if: always()
+        run: |
+          ./release/package/mgbuild.sh \
+          --toolchain $TOOLCHAIN \
+          --os $OS \
+          --arch $ARCH \
+          stop --remove

.github/workflows/stress_test_large.yaml

@@ -13,56 +13,63 @@ on:
         options:
           - Release
           - RelWithDebInfo
+      os:
+        type: choice
+        description: "Target OS on which the release tests will be run. Select 'all' if you want to package for every listed OS."
+        default: 'debian-11'
+        options:
+          - all
+          - amzn-2
+          - centos-7
+          - centos-9
+          - debian-11
+          - debian-12
+          - fedora-38
+          - fedora-39
+          - rocky-9.3
+          - ubuntu-20.04
+          - ubuntu-22.04
+      toolchain:
+        type: choice
+        description: "Toolchain version with which memgraph will be packaged."
+        default: 'v5'
+        options:
+          - v5
   push:
     tags:
       - "v*.*.*-rc*"
       - "v*.*-rc*"
   schedule:
     - cron: "0 22 * * *"
-
-env:
-  THREADS: 24
-  MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-  MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-  BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
-
+# ${{ (github.event.inputs.target_os == 'debian-10' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v4' }}
 jobs:
-  stress_test_large:
-    name: "Stress test large"
-    timeout-minutes: 720
+  StressTestLargeIndividual:
+    if: ${{ !github.event.inputs.os || github.event.inputs.os != 'all' }}
+    uses: ./.github/workflows/reusable_release_tests.yaml
+    with:
+      os: ${{ github.event.inputs.os || 'debian-11' }}
+      toolchain: ${{ github.event.inputs.toolchain || 'v5' }}
+      arch: "amd"
+      runner_arch_label: "X64"
+      threads: 24
+      build_type: ${{ github.event.inputs.build_type || 'Release' }}
+      run_stress_large: 'true'
+      run_release_tests: 'false'
+    secrets: inherit
+
+  StressTestLargeAll:
+    if: ${{ github.event.inputs.os == 'all' }}
     strategy:
       matrix:
-        os: [Debian10, Ubuntu20.04]
-        extra: [BigMemory, Gen8]
-        exclude:
-          - os: Debian10
-            extra: Gen8
-          - os: Ubuntu20.04
-            extra: BigMemory
-    runs-on: [self-hosted, Linux, X64, "${{ matrix.os }}", "${{ matrix.extra }}"]
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v4
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build release binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build release binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
-          make -j$THREADS
-
-      - name: Run stress test (large)
-        run: |
-          cd tests/stress
-          ./continuous_integration --large-dataset
+        os: [amzn-2, centos-7, centos-9, debian-11, debian-12, fedora-38, fedora-39, rocky-9.3, ubuntu-20.04, ubuntu-22.04]
+    uses: ./.github/workflows/reusable_release_tests.yaml
+    with:
+      os: ${{ matrix.os }}
+      toolchain: ${{ github.event.inputs.toolchain || 'v5' }}
+      arch: "amd"
+      runner_arch_label: "X64"
+      threads: 24
+      build_type: ${{ github.event.inputs.build_type || 'Release' }}
+      run_stress_large: 'true'
+      run_release_tests: 'false'
+    secrets: inherit

release/package/mgbuild.sh

@@ -435,6 +435,9 @@ test_memgraph() {
     stress-ssl)
       docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/stress && source ve3/bin/activate "'&& ./continuous_integration --use-ssl'
     ;;
+    stress-large)
+      docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/stress && source ve3/bin/activate "'&& ./continuous_integration --large-dataset'
+    ;;
     durability)
       docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/stress && source ve3/bin/activate "'&& python3 durability --num-steps 5'
     ;;