From 866ed45562d5ff7d77ad346c7c999eb4244c4029 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Budiseli=C4=87?= <marko.budiselic@memgraph.com>
Date: Tue, 16 Mar 2021 09:13:50 +0100
Subject: [PATCH] Add daily build for all major latest OSs (#110)
---
...lease_centos.yaml => release_centos8.yaml} | 15 +-
.../{release.yaml => release_debian10.yaml} | 10 +-
.github/workflows/release_ubuntu2004.yaml | 318 ++++++++++++++++++
3 files changed, 332 insertions(+), 11 deletions(-)
rename .github/workflows/{release_centos.yaml => release_centos8.yaml} (96%)
rename .github/workflows/{release.yaml => release_debian10.yaml} (97%)
create mode 100644 .github/workflows/release_ubuntu2004.yaml
diff --git a/.github/workflows/release_centos.yaml b/.github/workflows/release_centos8.yaml
similarity index 96%
rename from .github/workflows/release_centos.yaml
rename to .github/workflows/release_centos8.yaml
index 93dd8eaeb..49584f5bd 100644
--- a/.github/workflows/release_centos.yaml
+++ b/.github/workflows/release_centos8.yaml
@@ -1,11 +1,14 @@
-name: Release CentOS
+name: Release CentOS 8
-on: [workflow_dispatch]
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: "0 1 * * *"
jobs:
community_build:
name: "Community build"
- runs-on: [self-hosted, General, Linux, X64, CentOS7]
+ runs-on: [self-hosted, Linux, X64, CentOS8]
env:
THREADS: 24
timeout-minutes: 960
@@ -69,7 +72,7 @@ jobs:
coverage_build:
name: "Coverage build"
- runs-on: [self-hosted, General, Linux, X64, CentOS7]
+ runs-on: [self-hosted, Linux, X64, CentOS8]
env:
THREADS: 24
@@ -124,7 +127,7 @@ jobs:
debug_build:
name: "Debug build"
- runs-on: [self-hosted, General, Linux, X64, CentOS7]
+ runs-on: [self-hosted, Linux, X64, CentOS8]
env:
THREADS: 24
@@ -198,7 +201,7 @@ jobs:
release_build:
name: "Release build"
- runs-on: [self-hosted, General, Linux, X64, CentOS7]
+ runs-on: [self-hosted, Linux, X64, CentOS8]
env:
THREADS: 24
timeout-minutes: 960
diff --git a/.github/workflows/release.yaml b/.github/workflows/release_debian10.yaml
similarity index 97%
rename from .github/workflows/release.yaml
rename to .github/workflows/release_debian10.yaml
index fc7a2d5f0..7a728fef8 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release_debian10.yaml
@@ -1,4 +1,4 @@
-name: Release
+name: Release Debian10
on:
workflow_dispatch:
@@ -8,7 +8,7 @@ on:
jobs:
community_build:
name: "Community build"
- runs-on: [self-hosted, General, Linux, X64, Debian10]
+ runs-on: [self-hosted, Linux, X64, Debian10]
env:
THREADS: 24
timeout-minutes: 960
@@ -71,7 +71,7 @@ jobs:
coverage_build:
name: "Coverage build"
- runs-on: [self-hosted, General, Linux, X64, Debian10]
+ runs-on: [self-hosted, Linux, X64, Debian10]
env:
THREADS: 24
@@ -126,7 +126,7 @@ jobs:
debug_build:
name: "Debug build"
- runs-on: [self-hosted, General, Linux, X64, Debian10]
+ runs-on: [self-hosted, Linux, X64, Debian10]
env:
THREADS: 24
@@ -200,7 +200,7 @@ jobs:
release_build:
name: "Release build"
- runs-on: [self-hosted, General, Linux, X64, Debian10]
+ runs-on: [self-hosted, Linux, X64, Debian10]
env:
THREADS: 24
timeout-minutes: 960
diff --git a/.github/workflows/release_ubuntu2004.yaml b/.github/workflows/release_ubuntu2004.yaml
new file mode 100644
index 000000000..90386fce5
--- /dev/null
+++ b/.github/workflows/release_ubuntu2004.yaml
@@ -0,0 +1,318 @@
+name: Release Ubuntu20.04
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: "0 1 * * *"
+
+jobs:
+ community_build:
+ name: "Community build"
+ runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+ env:
+ THREADS: 24
+ timeout-minutes: 960
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build community binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v2/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build community binaries.
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
+ make -j$THREADS
+
+ - name: Create community DEB package
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v2/activate
+
+ # Create community DEB package.
+ cd build
+ mkdir output && cd output
+ cpack -G DEB --config ../CPackConfig.cmake
+
+ - name: Save community DEB package
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Community DEB package"
+ path: build/output/memgraph*.deb
+
+ - name: Run unit tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v2/activate
+
+ # Run unit tests.
+ cd build
+ ctest -R memgraph__unit --output-on-failure
+
+ - name: Run stress test (plain)
+ run: |
+ cd tests/stress
+ ./continuous_integration
+
+ - name: Run stress test (large)
+ run: |
+ cd tests/stress
+ ./continuous_integration --large-dataset
+
+ coverage_build:
+ name: "Coverage build"
+ runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+ env:
+ THREADS: 24
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build coverage binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v2/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build coverage binaries.
+ cd build
+ cmake -DTEST_COVERAGE=ON ..
+ make -j$THREADS memgraph__unit
+
+ - name: Run unit tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v2/activate
+
+ # Run unit tests.
+ cd build
+ ctest -R memgraph__unit --output-on-failure
+
+ - name: Compute code coverage
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v2/activate
+
+ # Compute code coverage.
+ cd tools/github
+ ./coverage_convert
+
+ # Package code coverage.
+ cd generated
+ tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
+
+ - name: Save code coverage
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Code coverage"
+ path: tools/github/generated/code_coverage.tar.gz
+
+ debug_build:
+ name: "Debug build"
+ runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+ env:
+ THREADS: 24
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build debug binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v2/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build debug binaries.
+ cd build
+ cmake ..
+ make -j$THREADS
+
+ - name: Run leftover CTest tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v2/activate
+
+ # Run leftover CTest tests (all except unit and benchmark tests).
+ cd build
+ ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
+
+ - name: Run drivers tests
+ run: |
+ ./tests/drivers/run.sh
+
+ - name: Run integration tests
+ run: |
+ cd tests/integration
+ for name in *; do
+ if [ ! -d $name ]; then continue; fi
+ pushd $name >/dev/null
+ echo "Running: $name"
+ if [ -x prepare.sh ]; then
+ ./prepare.sh
+ fi
+ if [ -x runner.py ]; then
+ ./runner.py
+ elif [ -x runner.sh ]; then
+ ./runner.sh
+ fi
+ echo
+ popd >/dev/null
+ done
+
+ - name: Run cppcheck and clang-format
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v2/activate
+
+ # Run cppcheck and clang-format.
+ cd tools/github
+ ./cppcheck_and_clang_format diff
+
+ - name: Save cppcheck and clang-format errors
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Code coverage"
+ path: tools/github/cppcheck_and_clang_format.txt
+
+ release_build:
+ name: "Release build"
+ runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+ env:
+ THREADS: 24
+ timeout-minutes: 960
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build release binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v2/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build release binaries.
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=release ..
+ make -j$THREADS
+
+ - name: Create enterprise DEB package
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v2/activate
+
+ # Create enterprise DEB package.
+ cd build
+ mkdir output && cd output
+ cpack -G DEB --config ../CPackConfig.cmake
+
+ - name: Save enterprise DEB package
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Enterprise DEB package"
+ path: build/output/memgraph*.deb
+
+ - name: Run micro benchmark tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v2/activate
+
+ # Run micro benchmark tests.
+ cd build
+ # The `eval` benchmark needs a large stack limit.
+ ulimit -s 262144
+ ctest -R memgraph__benchmark -V
+
+ - name: Run macro benchmark tests
+ run: |
+ cd tests/macro_benchmark
+ ./harness QuerySuite MemgraphRunner \
+ --groups aggregation 1000_create unwind_create dense_expand match \
+ --no-strict
+
+ - name: Run parallel macro benchmark tests
+ run: |
+ cd tests/macro_benchmark
+ ./harness QueryParallelSuite MemgraphRunner \
+ --groups aggregation_parallel create_parallel bfs_parallel \
+ --num-database-workers 9 --num-clients-workers 30 \
+ --no-strict
+
+ - name: Run GQL Behave tests
+ run: |
+ cd tests/gql_behave
+ ./continuous_integration
+
+ - name: Save quality assurance status
+ uses: actions/upload-artifact@v2
+ with:
+ name: "GQL Behave Status"
+ path: |
+ tests/gql_behave/gql_behave_status.csv
+ tests/gql_behave/gql_behave_status.html
+
+ - name: Run e2e replication tests
+ run: |
+ # TODO(gitbuda): Setup mgclient and pymgclient properly.
+ cd tests
+ ./setup.sh
+ source ve3/bin/activate
+ cd e2e
+ LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
+
+ - name: Run stress test (plain)
+ run: |
+ cd tests/stress
+ ./continuous_integration
+
+ - name: Run stress test (SSL)
+ run: |
+ cd tests/stress
+ ./continuous_integration --use-ssl
+
+ - name: Run stress test (large)
+ run: |
+ cd tests/stress
+ ./continuous_integration --large-dataset
+
+ - name: Run durability test (plain)
+ run: |
+ cd tests/stress
+ source ve3/bin/activate
+ python3 durability --num-steps 5
+
+ - name: Run durability test (large)
+ run: |
+ cd tests/stress
+ source ve3/bin/activate
+ python3 durability --num-steps 20