From f1484240a0c67050ff91a6f7ff4a667bac3212e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?=
<48765171+MarkoBarisic@users.noreply.github.com>
Date: Fri, 26 Jan 2024 17:01:25 +0100
Subject: [PATCH] Make release CI predictable (#1658)
* Move stress test large to a separate workflow
* Combine Debian10 and Ubuntu20.04 stress tests into one workflow
* Remove BigMemory tag from release_debian10
* Move e2e tests to a separate job
* Move debug interation tests to a separate job
* Move release durability and stress tests to a separate job
* Move release benchamarks to separate job
* Add 90 min timeout restriction to all jobs
* Move env variables to workflow level
* Move BUILD_TYPE env var to workflow level
---------
Co-authored-by: Aidar Samerkhanov <aidar.samerkhanov@memgraph.io>
---
.github/workflows/release_debian10.yaml | 194 +++++++++++++++-------
.github/workflows/release_ubuntu2004.yaml | 181 ++++++++++++++------
.github/workflows/stress_test_large.yaml | 62 +++++++
3 files changed, 331 insertions(+), 106 deletions(-)
create mode 100644 .github/workflows/stress_test_large.yaml
diff --git a/.github/workflows/release_debian10.yaml b/.github/workflows/release_debian10.yaml
index f22be163d..8fe3446a0 100644
--- a/.github/workflows/release_debian10.yaml
+++ b/.github/workflows/release_debian10.yaml
@@ -14,15 +14,17 @@ on:
schedule:
- cron: "0 22 * * *"
+env:
+ THREADS: 24
+ MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
+ MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+ BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
+
jobs:
community_build:
name: "Community build"
runs-on: [self-hosted, Linux, X64, Debian10]
- env:
- THREADS: 24
- MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
- MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
- timeout-minutes: 960
+ timeout-minutes: 90
steps:
- name: Set up repository
@@ -40,10 +42,6 @@ jobs:
# Initialize dependencies.
./init
- # Set default build_type to Release
- INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
- BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
-
# Build community binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DMG_ENTERPRISE=OFF ..
@@ -65,6 +63,7 @@ jobs:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+ timeout-minutes: 90
steps:
- name: Set up repository
@@ -118,10 +117,7 @@ jobs:
debug_build:
name: "Debug build"
runs-on: [self-hosted, Linux, X64, Debian10]
- env:
- THREADS: 24
- MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
- MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+ timeout-minutes: 90
steps:
- name: Set up repository
@@ -157,10 +153,6 @@ jobs:
run: |
./tests/drivers/run.sh
- - name: Run integration tests
- run: |
- tests/integration/run.sh
-
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
@@ -176,14 +168,40 @@ jobs:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
+ debug_integration_test:
+ name: "Debug integration tests"
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ timeout-minutes: 90
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v3
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build debug binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v4/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build debug binaries.
+ cd build
+ cmake ..
+ make -j$THREADS
+
+ - name: Run integration tests
+ run: |
+ tests/integration/run.sh
+
release_build:
name: "Release build"
- runs-on: [self-hosted, Linux, X64, Debian10, BigMemory]
- env:
- THREADS: 24
- MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
- MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
- timeout-minutes: 960
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ timeout-minutes: 90
steps:
- name: Set up repository
@@ -201,10 +219,6 @@ jobs:
# Initialize dependencies.
./init
- # Set default build_type to Release
- INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
- BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
-
# Build release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
@@ -231,6 +245,55 @@ jobs:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
+ - name: Run GQL Behave tests
+ run: |
+ cd tests
+ ./setup.sh /opt/toolchain-v4/activate
+ cd gql_behave
+ ./continuous_integration
+
+ - name: Save quality assurance status
+ uses: actions/upload-artifact@v3
+ with:
+ name: "GQL Behave Status"
+ path: |
+ tests/gql_behave/gql_behave_status.csv
+ tests/gql_behave/gql_behave_status.html
+
+ - name: Run unit tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v4/activate
+
+ # Run unit tests.
+ cd build
+ ctest -R memgraph__unit --output-on-failure
+
+ release_benchmark_tests:
+ name: "Release Benchmark Tests"
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ timeout-minutes: 90
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v3
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build release binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v4/activate
+ # Initialize dependencies.
+ ./init
+
+ # Build release binaries
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
+ make -j$THREADS
+
- name: Run micro benchmark tests
run: |
# Activate toolchain.
@@ -257,29 +320,30 @@ jobs:
--num-database-workers 9 --num-clients-workers 30 \
--no-strict
- - name: Run GQL Behave tests
- run: |
- cd tests
- ./setup.sh /opt/toolchain-v4/activate
- cd gql_behave
- ./continuous_integration
+ release_e2e_test:
+ name: "Release End-to-end Test"
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ timeout-minutes: 90
- - name: Save quality assurance status
- uses: actions/upload-artifact@v3
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v3
with:
- name: "GQL Behave Status"
- path: |
- tests/gql_behave/gql_behave_status.csv
- tests/gql_behave/gql_behave_status.html
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
- - name: Run unit tests
+ - name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
+ # Initialize dependencies.
+ ./init
- # Run unit tests.
+ # Build release binaries
cd build
- ctest -R memgraph__unit --output-on-failure
+ cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
+ make -j$THREADS
- name: Ensure Kafka and Pulsar are up
run: |
@@ -304,6 +368,32 @@ jobs:
cd ../pulsar
docker-compose down
+ release_durability_stress_tests:
+ name: "Release durability and stress tests"
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ timeout-minutes: 90
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v3
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build release binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v4/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build release binaries.
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
+ make -j$THREADS
+
- name: Run stress test (plain)
run: |
cd tests/stress
@@ -314,11 +404,6 @@ jobs:
cd tests/stress
./continuous_integration --use-ssl
- - name: Run stress test (large)
- run: |
- cd tests/stress
- ./continuous_integration --large-dataset
-
- name: Run durability test (plain)
run: |
cd tests/stress
@@ -334,11 +419,7 @@ jobs:
release_jepsen_test:
name: "Release Jepsen Test"
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
- env:
- THREADS: 24
- MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
- MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
- timeout-minutes: 60
+ timeout-minutes: 90
steps:
- name: Set up repository
@@ -355,15 +436,16 @@ jobs:
# Initialize dependencies.
./init
- # Set default build_type to Release
- INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
- BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
-
# Build only memgraph release binary.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS memgraph
+ - name: Refresh Jepsen Cluster
+ run: |
+ cd tests/jepsen
+ ./run.sh cluster-refresh
+
- name: Run Jepsen tests
run: |
cd tests/jepsen
diff --git a/.github/workflows/release_ubuntu2004.yaml b/.github/workflows/release_ubuntu2004.yaml
index c82d05ad1..ef5a5ab4a 100644
--- a/.github/workflows/release_ubuntu2004.yaml
+++ b/.github/workflows/release_ubuntu2004.yaml
@@ -14,15 +14,17 @@ on:
schedule:
- cron: "0 22 * * *"
+env:
+ THREADS: 24
+ MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
+ MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+ BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
+
jobs:
community_build:
name: "Community build"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
- env:
- THREADS: 24
- MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
- MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
- timeout-minutes: 960
+ timeout-minutes: 90
steps:
- name: Set up repository
@@ -40,10 +42,6 @@ jobs:
# Initialize dependencies.
./init
- # Set default build_type to Release
- INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
- BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
-
# Build community binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DMG_ENTERPRISE=OFF ..
@@ -61,10 +59,7 @@ jobs:
coverage_build:
name: "Coverage build"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
- env:
- THREADS: 24
- MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
- MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+ timeout-minutes: 90
steps:
- name: Set up repository
@@ -118,10 +113,7 @@ jobs:
debug_build:
name: "Debug build"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
- env:
- THREADS: 24
- MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
- MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+ timeout-minutes: 90
steps:
- name: Set up repository
@@ -157,10 +149,6 @@ jobs:
run: |
./tests/drivers/run.sh
- - name: Run integration tests
- run: |
- tests/integration/run.sh
-
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
@@ -176,14 +164,40 @@ jobs:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
+ debug_integration_test:
+ name: "Debug integration tests"
+ runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+ timeout-minutes: 90
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v3
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build debug binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v4/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build debug binaries.
+ cd build
+ cmake ..
+ make -j$THREADS
+
+ - name: Run integration tests
+ run: |
+ tests/integration/run.sh
+
release_build:
name: "Release build"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
- env:
- THREADS: 24
- MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
- MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
- timeout-minutes: 960
+ timeout-minutes: 90
steps:
- name: Set up repository
@@ -201,10 +215,6 @@ jobs:
# Initialize dependencies.
./init
- # Set default build_type to Release
- INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
- BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
-
# Build release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
@@ -231,6 +241,55 @@ jobs:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
+ - name: Run GQL Behave tests
+ run: |
+ cd tests
+ ./setup.sh /opt/toolchain-v4/activate
+ cd gql_behave
+ ./continuous_integration
+
+ - name: Save quality assurance status
+ uses: actions/upload-artifact@v3
+ with:
+ name: "GQL Behave Status"
+ path: |
+ tests/gql_behave/gql_behave_status.csv
+ tests/gql_behave/gql_behave_status.html
+
+ - name: Run unit tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v4/activate
+
+ # Run unit tests.
+ cd build
+ ctest -R memgraph__unit --output-on-failure
+
+ release_benchmark_tests:
+ name: "Release Benchmark Tests"
+ runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+ timeout-minutes: 90
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v3
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build release binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v4/activate
+ # Initialize dependencies.
+ ./init
+
+ # Build release binaries
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
+ make -j$THREADS
+
- name: Run micro benchmark tests
run: |
# Activate toolchain.
@@ -257,29 +316,30 @@ jobs:
--num-database-workers 9 --num-clients-workers 30 \
--no-strict
- - name: Run GQL Behave tests
- run: |
- cd tests
- ./setup.sh /opt/toolchain-v4/activate
- cd gql_behave
- ./continuous_integration
+ release_e2e_test:
+ name: "Release End-to-end Test"
+ runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+ timeout-minutes: 90
- - name: Save quality assurance status
- uses: actions/upload-artifact@v3
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v3
with:
- name: "GQL Behave Status"
- path: |
- tests/gql_behave/gql_behave_status.csv
- tests/gql_behave/gql_behave_status.html
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
- - name: Run unit tests
+ - name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
+ # Initialize dependencies.
+ ./init
- # Run unit tests.
+ # Build release binaries
cd build
- ctest -R memgraph__unit --output-on-failure
+ cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
+ make -j$THREADS
- name: Ensure Kafka and Pulsar are up
run: |
@@ -304,6 +364,32 @@ jobs:
cd ../pulsar
docker-compose down
+ release_durability_stress_tests:
+ name: "Release durability and stress tests"
+ runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+ timeout-minutes: 90
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v3
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build release binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v4/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build release binaries.
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
+ make -j$THREADS
+
- name: Run stress test (plain)
run: |
cd tests/stress
@@ -314,11 +400,6 @@ jobs:
cd tests/stress
./continuous_integration --use-ssl
- - name: Run stress test (large)
- run: |
- cd tests/stress
- ./continuous_integration --large-dataset
-
- name: Run durability test (plain)
run: |
cd tests/stress
diff --git a/.github/workflows/stress_test_large.yaml b/.github/workflows/stress_test_large.yaml
new file mode 100644
index 000000000..0531d6e56
--- /dev/null
+++ b/.github/workflows/stress_test_large.yaml
@@ -0,0 +1,62 @@
+name: Stress test large
+
+on:
+ workflow_dispatch:
+ inputs:
+ build_type:
+ type: choice
+ description: "Memgraph Build type. Default value is Release."
+ default: 'Release'
+ options:
+ - Release
+ - RelWithDebInfo
+
+ schedule:
+ - cron: "0 22 * * *"
+
+env:
+ THREADS: 24
+ MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
+ MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+ BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
+
+jobs:
+ stress_test_large:
+ name: "Stress test large"
+ timeout-minutes: 600
+ strategy:
+ matrix:
+ os: [Debian10, Ubuntu20.04]
+ extra: [BigMemory, Gen8]
+ exclude:
+ - os: Debian10
+ extra: Gen8
+ - os: Ubuntu20.04
+ extra: BigMemory
+ runs-on: [self-hosted, Linux, X64, "${{ matrix.os }}", "${{ matrix.extra }}"]
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v3
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build release binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v4/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build release binaries.
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
+ make -j$THREADS
+
+ - name: Run stress test (large)
+ run: |
+ cd tests/stress
+ ./continuous_integration --large-dataset