From 0dcfdb9b89217d563a740185be9a65bab4dabf4d Mon Sep 17 00:00:00 2001
From: Matej Ferencevic <matej.ferencevic@memgraph.io>
Date: Mon, 21 Sep 2020 12:22:40 +0200
Subject: [PATCH] Add GitHub workflows
---
.github/workflows/diff.yaml | 284 ++++++++++++++++
.github/workflows/release.yaml | 307 ++++++++++++++++++
.github/workflows/release_centos.yaml | 306 +++++++++++++++++
init | 122 ++++---
tests/drivers/javascript/run.sh | 6 +-
tests/drivers/run.sh | 5 +-
tests/integration/transactions/runner.sh | 2 +-
tests/qa/.gitignore | 2 +-
tests/qa/continuous_integration | 6 +-
tests/unit/CMakeLists.txt | 6 +-
tools/github/.gitignore | 2 +
tools/{apollo => github}/coverage_convert | 26 +-
.../{apollo => github}/coverage_parse_export | 0
.../cppcheck_and_clang_format} | 12 +-
.../macro_benchmark_summary | 0
15 files changed, 1011 insertions(+), 75 deletions(-)
create mode 100644 .github/workflows/diff.yaml
create mode 100644 .github/workflows/release.yaml
create mode 100644 .github/workflows/release_centos.yaml
create mode 100644 tools/github/.gitignore
rename tools/{apollo => github}/coverage_convert (57%)
rename tools/{apollo => github}/coverage_parse_export (100%)
rename tools/{apollo/cppcheck => github/cppcheck_and_clang_format} (91%)
rename tools/{apollo => github}/macro_benchmark_summary (100%)
diff --git a/.github/workflows/diff.yaml b/.github/workflows/diff.yaml
new file mode 100644
index 000000000..cac272151
--- /dev/null
+++ b/.github/workflows/diff.yaml
@@ -0,0 +1,284 @@
+name: Diff
+
+on: [push]
+
+jobs:
+ community_build:
+ name: "Community build"
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ env:
+ THREADS: 24
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build community binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build community binaries.
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
+ make -j$THREADS
+
+ - name: Run unit tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run unit tests.
+ cd build
+ ctest -R memgraph__unit --output-on-failure
+
+ - name: Run stress test
+ run: |
+ cd tests/stress
+ ./continuous_integration
+
+ coverage_build:
+ name: "Coverage build"
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ env:
+ THREADS: 24
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build coverage binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build coverage binaries.
+ cd build
+ cmake -DTEST_COVERAGE=ON ..
+ make -j$THREADS memgraph__unit
+
+ - name: Run unit tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run unit tests.
+ cd build
+ ctest -R memgraph__unit --output-on-failure
+
+ - name: Compute code coverage
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Compute code coverage.
+ cd tools/github
+ ./coverage_convert
+
+ # Package code coverage.
+ cd generated
+ tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
+
+ - name: Save code coverage
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Code coverage"
+ path: tools/github/generated/code_coverage.tar.gz
+
+ debug_build:
+ name: "Debug build"
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ env:
+ THREADS: 24
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build debug binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build debug binaries.
+ cd build
+ cmake ..
+ make -j$THREADS
+
+ - name: Run leftover CTest tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run leftover CTest tests (all except unit and benchmark tests).
+ cd build
+ ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
+
+ - name: Run drivers tests
+ run: |
+ ./tests/drivers/run.sh
+
+ - name: Run integration tests
+ run: |
+ cd tests/integration
+ for name in *; do
+ if [ ! -d $name ]; then continue; fi
+ pushd $name >/dev/null
+ echo "Running: $name"
+ if [ -x prepare.sh ]; then
+ ./prepare.sh
+ fi
+ if [ -x runner.py ]; then
+ ./runner.py
+ elif [ -x runner.sh ]; then
+ ./runner.sh
+ fi
+ echo
+ popd >/dev/null
+ done
+
+ - name: Run cppcheck and clang-format
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run cppcheck and clang-format.
+ cd tools/github
+ ./cppcheck_and_clang_format diff
+
+ - name: Save cppcheck and clang-format errors
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Code coverage"
+ path: tools/github/cppcheck_and_clang_format.txt
+
+ release_build:
+ name: "Release build"
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ env:
+ THREADS: 24
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Set up parent
+ run: |
+ # Remove parent folder (if it exists).
+ cd ..
+ if [ -d parent ]; then
+ rm -rf parent
+ fi
+
+ # Copy untouched repository to parent folder.
+ cp -r memgraph parent
+
+ # Checkout previous commit
+ cd parent
+ git checkout HEAD~1
+
+ - name: Build release binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build release binaries.
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=release ..
+ make -j$THREADS
+
+ - name: Build parent binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ cd ../parent
+ ./init
+
+ # Build parent binaries.
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=release ..
+ make -j$THREADS memgraph memgraph__macro_benchmark
+
+ - name: Run macro benchmark tests
+ run: |
+ cd tests/macro_benchmark
+ ./harness QuerySuite MemgraphRunner \
+ --groups aggregation 1000_create unwind_create dense_expand match \
+ --no-strict
+
+ - name: Run parent macro benchmark tests
+ run: |
+ cd ../parent/tests/macro_benchmark
+ ./harness QuerySuite MemgraphRunner \
+ --groups aggregation 1000_create unwind_create dense_expand match \
+ --no-strict
+
+ - name: Compute macro benchmark summary
+ run: |
+ ./tools/github/macro_benchmark_summary \
+ --current tests/macro_benchmark/.harness_summary \
+ --previous ../parent/tests/macro_benchmark/.harness_summary \
+ --output macro_benchmark_summary.txt
+
+ - name: Save macro benchmark summary
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Macro benchmark summary"
+ path: macro_benchmark_summary.txt
+
+ - name: Run quality assurance tests
+ run: |
+ cd tests/qa
+ ./continuous_integration
+
+ - name: Save quality assurance status
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Quality assurance status"
+ path: tests/qa/quality_assurance_status.txt
+
+ - name: Run stress test (plain)
+ run: |
+ cd tests/stress
+ ./continuous_integration
+
+ - name: Run stress test (SSL)
+ run: |
+ cd tests/stress
+ ./continuous_integration --use-ssl
+
+ - name: Run durability test
+ run: |
+ cd tests/stress
+ source ve3/bin/activate
+ python3 durability --num-steps 5
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
new file mode 100644
index 000000000..db4da80a0
--- /dev/null
+++ b/.github/workflows/release.yaml
@@ -0,0 +1,307 @@
+name: Release
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: "0 1 * * *"
+
+jobs:
+ community_build:
+ name: "Community build"
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ env:
+ THREADS: 24
+ timeout-minutes: 960
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build community binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build community binaries.
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
+ make -j$THREADS
+
+ - name: Create community DEB package
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Create community DEB package.
+ cd build
+ mkdir output && cd output
+ cpack -G DEB --config ../CPackConfig.cmake
+
+ - name: Save community DEB package
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Community DEB package"
+ path: build/output/memgraph*.deb
+
+ - name: Run unit tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run unit tests.
+ cd build
+ ctest -R memgraph__unit --output-on-failure
+
+ - name: Run stress test (plain)
+ run: |
+ cd tests/stress
+ ./continuous_integration
+
+ - name: Run stress test (large)
+ run: |
+ cd tests/stress
+ ./continuous_integration --large-dataset
+
+ coverage_build:
+ name: "Coverage build"
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ env:
+ THREADS: 24
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build coverage binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build coverage binaries.
+ cd build
+ cmake -DTEST_COVERAGE=ON ..
+ make -j$THREADS memgraph__unit
+
+ - name: Run unit tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run unit tests.
+ cd build
+ ctest -R memgraph__unit --output-on-failure
+
+ - name: Compute code coverage
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Compute code coverage.
+ cd tools/github
+ ./coverage_convert
+
+ # Package code coverage.
+ cd generated
+ tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
+
+ - name: Save code coverage
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Code coverage"
+ path: tools/github/generated/code_coverage.tar.gz
+
+ debug_build:
+ name: "Debug build"
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ env:
+ THREADS: 24
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build debug binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build debug binaries.
+ cd build
+ cmake ..
+ make -j$THREADS
+
+ - name: Run leftover CTest tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run leftover CTest tests (all except unit and benchmark tests).
+ cd build
+ ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
+
+ - name: Run drivers tests
+ run: |
+ ./tests/drivers/run.sh
+
+ - name: Run integration tests
+ run: |
+ cd tests/integration
+ for name in *; do
+ if [ ! -d $name ]; then continue; fi
+ pushd $name >/dev/null
+ echo "Running: $name"
+ if [ -x prepare.sh ]; then
+ ./prepare.sh
+ fi
+ if [ -x runner.py ]; then
+ ./runner.py
+ elif [ -x runner.sh ]; then
+ ./runner.sh
+ fi
+ echo
+ popd >/dev/null
+ done
+
+ - name: Run cppcheck and clang-format
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run cppcheck and clang-format.
+ cd tools/github
+ ./cppcheck_and_clang_format diff
+
+ - name: Save cppcheck and clang-format errors
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Code coverage"
+ path: tools/github/cppcheck_and_clang_format.txt
+
+ release_build:
+ name: "Release build"
+ runs-on: [self-hosted, Linux, X64, Debian10]
+ env:
+ THREADS: 24
+ timeout-minutes: 960
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build release binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build release binaries.
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=release ..
+ make -j$THREADS
+
+ - name: Create enterprise DEB package
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Create enterprise DEB package.
+ cd build
+ mkdir output && cd output
+ cpack -G DEB --config ../CPackConfig.cmake
+
+ - name: Save enterprise DEB package
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Enterprise DEB package"
+ path: build/output/memgraph*.deb
+
+ - name: Run micro benchmark tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run micro benchmark tests.
+ cd build
+ # The `eval` benchmark needs a large stack limit.
+ ulimit -s 262144
+ ctest -R memgraph__benchmark -V
+
+ - name: Run macro benchmark tests
+ run: |
+ cd tests/macro_benchmark
+ ./harness QuerySuite MemgraphRunner \
+ --groups aggregation 1000_create unwind_create dense_expand match \
+ --no-strict
+
+ - name: Run parallel macro benchmark tests
+ run: |
+ cd tests/macro_benchmark
+ ./harness QueryParallelSuite MemgraphRunner \
+ --groups aggregation_parallel create_parallel bfs_parallel \
+ --num-database-workers 9 --num-clients-workers 30 \
+ --no-strict
+
+ - name: Run quality assurance tests
+ run: |
+ cd tests/qa
+ ./continuous_integration
+
+ - name: Save quality assurance status
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Quality assurance status"
+ path: tests/qa/quality_assurance_status.txt
+
+ - name: Run stress test (plain)
+ run: |
+ cd tests/stress
+ ./continuous_integration
+
+ - name: Run stress test (SSL)
+ run: |
+ cd tests/stress
+ ./continuous_integration --use-ssl
+
+ - name: Run stress test (large)
+ run: |
+ cd tests/stress
+ ./continuous_integration --large-dataset
+
+ - name: Run durability test (plain)
+ run: |
+ cd tests/stress
+ source ve3/bin/activate
+ python3 durability --num-steps 5
+
+ - name: Run durability test (large)
+ run: |
+ cd tests/stress
+ source ve3/bin/activate
+ python3 durability --num-steps 20
diff --git a/.github/workflows/release_centos.yaml b/.github/workflows/release_centos.yaml
new file mode 100644
index 000000000..ae5009d6a
--- /dev/null
+++ b/.github/workflows/release_centos.yaml
@@ -0,0 +1,306 @@
+name: Release CentOS
+
+on: [workflow_dispatch]
+
+jobs:
+ community_build:
+ name: "Community build"
+ runs-on: [self-hosted, Linux, X64, CentOS7]
+ env:
+ THREADS: 24
+ timeout-minutes: 960
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build community binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build community binaries.
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
+ make -j$THREADS
+
+ - name: Create community RPM package
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Create community RPM package.
+ cd build
+ mkdir output && cd output
+ cpack -G RPM --config ../CPackConfig.cmake
+ rpmlint memgraph*.rpm
+
+ - name: Save community RPM package
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Community RPM package"
+ path: build/output/memgraph*.rpm
+
+ - name: Run unit tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run unit tests.
+ cd build
+ ctest -R memgraph__unit --output-on-failure
+
+ - name: Run stress test (plain)
+ run: |
+ cd tests/stress
+ ./continuous_integration
+
+ - name: Run stress test (large)
+ run: |
+ cd tests/stress
+ ./continuous_integration --large-dataset
+
+ coverage_build:
+ name: "Coverage build"
+ runs-on: [self-hosted, Linux, X64, CentOS7]
+ env:
+ THREADS: 24
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build coverage binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build coverage binaries.
+ cd build
+ cmake -DTEST_COVERAGE=ON ..
+ make -j$THREADS memgraph__unit
+
+ - name: Run unit tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run unit tests.
+ cd build
+ ctest -R memgraph__unit --output-on-failure
+
+ - name: Compute code coverage
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Compute code coverage.
+ cd tools/github
+ ./coverage_convert
+
+ # Package code coverage.
+ cd generated
+ tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
+
+ - name: Save code coverage
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Code coverage"
+ path: tools/github/generated/code_coverage.tar.gz
+
+ debug_build:
+ name: "Debug build"
+ runs-on: [self-hosted, Linux, X64, CentOS7]
+ env:
+ THREADS: 24
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build debug binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build debug binaries.
+ cd build
+ cmake ..
+ make -j$THREADS
+
+ - name: Run leftover CTest tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run leftover CTest tests (all except unit and benchmark tests).
+ cd build
+ ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
+
+ - name: Run drivers tests
+ run: |
+ ./tests/drivers/run.sh
+
+ - name: Run integration tests
+ run: |
+ cd tests/integration
+ for name in *; do
+ if [ ! -d $name ]; then continue; fi
+ pushd $name >/dev/null
+ echo "Running: $name"
+ if [ -x prepare.sh ]; then
+ ./prepare.sh
+ fi
+ if [ -x runner.py ]; then
+ ./runner.py
+ elif [ -x runner.sh ]; then
+ ./runner.sh
+ fi
+ echo
+ popd >/dev/null
+ done
+
+ - name: Run cppcheck and clang-format
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run cppcheck and clang-format.
+ cd tools/github
+ ./cppcheck_and_clang_format diff
+
+ - name: Save cppcheck and clang-format errors
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Code coverage"
+ path: tools/github/cppcheck_and_clang_format.txt
+
+ release_build:
+ name: "Release build"
+ runs-on: [self-hosted, Linux, X64, CentOS7]
+ env:
+ THREADS: 24
+ timeout-minutes: 960
+
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v2
+ with:
+ # Number of commits to fetch. `0` indicates all history for all
+ # branches and tags. (default: 1)
+ fetch-depth: 0
+
+ - name: Build release binaries
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Initialize dependencies.
+ ./init
+
+ # Build release binaries.
+ cd build
+ cmake -DCMAKE_BUILD_TYPE=release ..
+ make -j$THREADS
+
+ - name: Create enterprise RPM package
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Create enterprise RPM package.
+ cd build
+ mkdir output && cd output
+ cpack -G RPM --config ../CPackConfig.cmake
+ rpmlint memgraph*.rpm
+
+ - name: Save enterprise RPM package
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Enterprise RPM package"
+ path: build/output/memgraph*.rpm
+
+ - name: Run micro benchmark tests
+ run: |
+ # Activate toolchain.
+ source /opt/toolchain-v1/activate
+
+ # Run micro benchmark tests.
+ cd build
+ # The `eval` benchmark needs a large stack limit.
+ ulimit -s 262144
+ ctest -R memgraph__benchmark -V
+
+ - name: Run macro benchmark tests
+ run: |
+ cd tests/macro_benchmark
+ ./harness QuerySuite MemgraphRunner \
+ --groups aggregation 1000_create unwind_create dense_expand match \
+ --no-strict
+
+ - name: Run parallel macro benchmark tests
+ run: |
+ cd tests/macro_benchmark
+ ./harness QueryParallelSuite MemgraphRunner \
+ --groups aggregation_parallel create_parallel bfs_parallel \
+ --num-database-workers 9 --num-clients-workers 30 \
+ --no-strict
+
+ - name: Run quality assurance tests
+ run: |
+ cd tests/qa
+ ./continuous_integration
+
+ - name: Save quality assurance status
+ uses: actions/upload-artifact@v2
+ with:
+ name: "Quality assurance status"
+ path: tests/qa/quality_assurance_status.txt
+
+ - name: Run stress test (plain)
+ run: |
+ cd tests/stress
+ ./continuous_integration
+
+ - name: Run stress test (SSL)
+ run: |
+ cd tests/stress
+ ./continuous_integration --use-ssl
+
+ - name: Run stress test (large)
+ run: |
+ cd tests/stress
+ ./continuous_integration --large-dataset
+
+ - name: Run durability test (plain)
+ run: |
+ cd tests/stress
+ source ve3/bin/activate
+ python3 durability --num-steps 5
+
+ - name: Run durability test (large)
+ run: |
+ cd tests/stress
+ source ve3/bin/activate
+ python3 durability --num-steps 20
diff --git a/init b/init
index b6f4033f2..795874fdd 100755
--- a/init
+++ b/init
@@ -1,22 +1,44 @@
#!/bin/bash -e
-required_pkgs=(git # source code control
- make pkg-config # build system
- curl wget # for downloading libs
- uuid-dev default-jre-headless # required by antlr
- libreadline-dev # for memgraph console
- libssl-dev
- libseccomp-dev
- python3 python-virtualenv python3-pip # for qa, macro_benchmark and stress tests
- python3-yaml # for the configuration generator
- libcurl4-openssl-dev # mg-requests
- sbcl # for custom Lisp C++ preprocessing
- )
+required_pkgs_apt=(git # source code control
+ make pkg-config # build system
+ curl wget # for downloading libs
+ uuid-dev default-jre-headless # required by antlr
+ libreadline-dev # for memgraph console
+ libpython3-dev python3-dev # for query modules
+ libssl-dev
+ libseccomp-dev
+ python3 python-virtualenv python3-pip # for qa, macro_benchmark and stress tests
+ python3-yaml # for the configuration generator
+ libcurl4-openssl-dev # mg-requests
+ sbcl # for custom Lisp C++ preprocessing
+ )
-optional_pkgs=(doxygen graphviz # source documentation generators
- php-cli # for user technical documentation generators
- mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests
- )
+optional_pkgs_apt=(doxygen graphviz # source documentation generators
+ php-cli # for user technical documentation generators
+ mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests
+ )
+
+
+required_pkgs_yum=(# NOTE: git is too old on CentOS, install it manually from IUS
+ make pkgconfig # build system
+ curl wget # for downloading libs
+ libuuid-devel java-1.8.0-openjdk # required by antlr
+ readline-devel # for memgraph console
+ python3-devel # for query modules
+ openssl-devel
+ libseccomp-devel
+ python3 python-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
+ # NOTE: python3-yaml doesn't exist on CentOS, install it manually using `pip3 install PyYAML`
+ libcurl-devel # mg-requests
+ sbcl # for custom Lisp C++ preprocessing
+ rpm-build rpmlint # for RPM package building
+ )
+
+optional_pkgs_yum=(doxygen graphviz # source documentation generators
+ php-cli # for user technical documentation generators
+ mono-complete nodejs zip unzip java-1.8.0-openjdk-devel # for driver tests
+ )
use_sudo=0
@@ -76,29 +98,55 @@ echo "Started installing dependencies for Memgraph"
required_missing=0
-# install all dependencies on debian based operating systems
-for pkg in ${required_pkgs[@]}; do
- if dpkg -s $pkg 2>/dev/null >/dev/null; then
- echo "Found $pkg"
- elif (( $use_sudo )); then
- echo "Installing $pkg"
- if [[ ! `sudo apt-get -y install $pkg` ]]; then
- echo "Didn't install $pkg [required]"
+# install all dependencies on supported operating systems
+if command -v apt-get >/dev/null; then
+ for pkg in ${required_pkgs_apt[@]}; do
+ if dpkg -s $pkg 2>/dev/null >/dev/null; then
+ echo "Found $pkg"
+ elif (( $use_sudo )); then
+ echo "Installing $pkg"
+ if [[ ! `sudo apt-get -y install $pkg` ]]; then
+ echo "Didn't install $pkg [required]"
+ required_missing=1
+ fi
+ else
+ echo "Missing $pkg [required]"
required_missing=1
fi
- else
- echo "Missing $pkg [required]"
- required_missing=1
- fi
-done
-
-for pkg in ${optional_pkgs[@]}; do
- if dpkg -s $pkg 2>/dev/null >/dev/null; then
- echo "Found $pkg [optional]"
- else
- echo "Missing $pkg [optional]"
- fi
-done
+ done
+ for pkg in ${optional_pkgs_apt[@]}; do
+ if dpkg -s $pkg 2>/dev/null >/dev/null; then
+ echo "Found $pkg [optional]"
+ else
+ echo "Missing $pkg [optional]"
+ fi
+ done
+elif command -v yum >/dev/null; then
+ for pkg in ${required_pkgs_yum[@]}; do
+ if yum list installed $pkg 2>/dev/null >/dev/null; then
+ echo "Found $pkg"
+ elif (( $use_sudo )); then
+ echo "Installing $pkg"
+ if [[ ! `sudo yum install -y $pkg` ]]; then
+ echo "Didn't install $pkg [required]"
+ required_missing=1
+ fi
+ else
+ echo "Missing $pkg [required]"
+ required_missing=1
+ fi
+ done
+ for pkg in ${optional_pkgs_yum[@]}; do
+ if yum list installed $pkg 2>/dev/null >/dev/null; then
+ echo "Found $pkg [optional]"
+ else
+ echo "Missing $pkg [optional]"
+ fi
+ done
+else
+ echo "Unsupported distribution!"
+ exit 1
+fi
if (( $required_missing )); then
echo "Missing required packages. EXITING!"
diff --git a/tests/drivers/javascript/run.sh b/tests/drivers/javascript/run.sh
index f003a5175..4528f4e4f 100755
--- a/tests/drivers/javascript/run.sh
+++ b/tests/drivers/javascript/run.sh
@@ -3,7 +3,7 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$DIR"
-if ! which nodejs >/dev/null; then
+if ! which node >/dev/null; then
echo "Please install nodejs!"
exit 1
fi
@@ -15,5 +15,5 @@ if [ ! -d node_modules ]; then
rm driver.tar.gz || exit 1
fi
-nodejs basic.js
-nodejs max_query_length.js
+node basic.js
+node max_query_length.js
diff --git a/tests/drivers/run.sh b/tests/drivers/run.sh
index c98799b0a..0243e7e2b 100755
--- a/tests/drivers/run.sh
+++ b/tests/drivers/run.sh
@@ -31,7 +31,8 @@ fi
$binary_dir/memgraph \
--data-directory=$tmpdir \
--query-execution-timeout-sec=5 \
- --bolt-session-inactivity-timeout=10 &
+ --bolt-session-inactivity-timeout=10 \
+ --min-log-level 1 &
pid=$!
wait_for_server 7687
@@ -53,7 +54,7 @@ done
# Stop memgraph.
kill $pid
-wait -n
+wait $pid
code_mg=$?
# Temporary directory cleanup.
diff --git a/tests/integration/transactions/runner.sh b/tests/integration/transactions/runner.sh
index 10195fb59..432907630 100755
--- a/tests/integration/transactions/runner.sh
+++ b/tests/integration/transactions/runner.sh
@@ -23,7 +23,7 @@ code=$?
# Shutdown the memgraph process.
kill $pid
-wait -n
+wait $pid
code_mg=$?
# Check memgraph exit code.
diff --git a/tests/qa/.gitignore b/tests/qa/.gitignore
index 388af07eb..26a36bb39 100644
--- a/tests/qa/.gitignore
+++ b/tests/qa/.gitignore
@@ -4,4 +4,4 @@
*~
*.pyc
ve3/
-.quality_assurance_status
+quality_assurance_status.txt
diff --git a/tests/qa/continuous_integration b/tests/qa/continuous_integration
index b88b48da5..29b12cba7 100755
--- a/tests/qa/continuous_integration
+++ b/tests/qa/continuous_integration
@@ -27,7 +27,7 @@ import yaml
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
TESTS_DIR = os.path.join(SCRIPT_DIR, "tests")
BASE_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", ".."))
-BUILD_DIR = os.path.join(BASE_DIR, "build_release")
+BUILD_DIR = os.path.join(BASE_DIR, "build")
def wait_for_server(port, delay=0.01):
@@ -131,7 +131,7 @@ def main():
# Load tests from config file
with open(os.path.join(TESTS_DIR, "config.yaml")) as f:
- suites = yaml.load(f)
+ suites = yaml.safe_load(f)
# venv used to run the qa engine
venv_python = os.path.join(SCRIPT_DIR, "ve3", "bin", "python3")
@@ -184,7 +184,7 @@ def main():
qa_status_message = generate_remarkup(status_data)
# Create the report file
- qa_status_path = os.path.join(SCRIPT_DIR, ".quality_assurance_status")
+ qa_status_path = os.path.join(SCRIPT_DIR, "quality_assurance_status.txt")
with open(qa_status_path, "w") as f:
f.write(qa_status_message)
diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index 4bb88f5e0..db97cb2d5 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -13,7 +13,11 @@ function(add_unit_test test_cpp)
set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name})
target_link_libraries(${target_name} gtest gmock gtest_main Threads::Threads)
# register test
- add_test(${target_name} ${exec_name})
+ if(TEST_COVERAGE)
+ add_test(${target_name} env LLVM_PROFILE_FILE=${exec_name}.profraw ./${exec_name})
+ else()
+ add_test(${target_name} ${exec_name})
+ endif()
# add to memgraph__unit target
add_dependencies(memgraph__unit ${target_name})
endfunction(add_unit_test)
diff --git a/tools/github/.gitignore b/tools/github/.gitignore
new file mode 100644
index 000000000..8c743badd
--- /dev/null
+++ b/tools/github/.gitignore
@@ -0,0 +1,2 @@
+cppcheck_and_clang_format.txt
+generated/*
diff --git a/tools/apollo/coverage_convert b/tools/github/coverage_convert
similarity index 57%
rename from tools/apollo/coverage_convert
rename to tools/github/coverage_convert
index 15280f747..232ef697d 100755
--- a/tools/apollo/coverage_convert
+++ b/tools/github/coverage_convert
@@ -2,13 +2,15 @@
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
project_dir="$( dirname "$( dirname "$script_dir" )" )"
+source_dir="$project_dir/src"
+unit_tests_dir="$project_dir/build/tests/unit"
generated_dir="$script_dir/generated"
html_dir="$generated_dir/html"
data_file="$generated_dir/default.profdata"
json_file="$generated_dir/report.json"
coverage_file="$generated_dir/coverage.json"
-summary_file="$generated_dir/.coverage_summary"
+summary_file="$generated_dir/summary.rmu"
# cleanup output directory
if [ -d "$generated_dir" ]; then
@@ -17,22 +19,14 @@ fi
mkdir "$generated_dir"
# merge raw coverage info
-raw_files="$( find "$HOME" -name "*.profraw" | tr '\n' ' ' )"
+raw_files="$( find "$unit_tests_dir" -name "*.profraw" | sort | tr '\n' ' ' )"
llvm-profdata merge -sparse $raw_files -o "$data_file"
# create list of binaries
-cnt=0
-obj_files=""
-for prof_file in $raw_files; do
- if [ $cnt -gt 0 ]; then
- obj_files+=" -object "
- fi
- obj_files+="$( find "$( dirname "$prof_file" )" -executable -type f | head -n 1 )"
- cnt=$((cnt + 1))
-done
+obj_files="$( echo "$raw_files" | sed "s/\.profraw//g" | sed -r 's/ +$//g' | sed "s/ / -object /g" )"
# create list of source files
-src_files=$( find "$HOME" \( -name '*.cpp' -o -name '*.hpp' \) -print | sort | tr '\n' ' ' )
+src_files=$( find "$source_dir" \( -name '*.cpp' -o -name '*.hpp' \) -print | sort | tr '\n' ' ' )
# generate html output
llvm-cov show $obj_files \
@@ -43,14 +37,6 @@ llvm-cov show $obj_files \
-Xdemangler c++filt -Xdemangler -n \
$src_files
-# fix names in html output
-coverage_dir="$html_dir/coverage"
-mv $coverage_dir/workspace/memgraph/* $html_dir/coverage/
-rm -r $coverage_dir/workspace
-find $coverage_dir -name "*.html" -exec sed -i 's@/workspace/memgraph/@@g' {} \;
-find $coverage_dir -name "*.html" -exec sed -i 's@../../style.css@style.css@g' {} \;
-sed -i 's@/workspace/memgraph@@g' $html_dir/index.html
-
# generate json output
llvm-cov export $obj_files \
-instr-profile "$data_file" \
diff --git a/tools/apollo/coverage_parse_export b/tools/github/coverage_parse_export
similarity index 100%
rename from tools/apollo/coverage_parse_export
rename to tools/github/coverage_parse_export
diff --git a/tools/apollo/cppcheck b/tools/github/cppcheck_and_clang_format
similarity index 91%
rename from tools/apollo/cppcheck
rename to tools/github/cppcheck_and_clang_format
index 188af1d4f..044fb270b 100755
--- a/tools/apollo/cppcheck
+++ b/tools/github/cppcheck_and_clang_format
@@ -3,20 +3,18 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$DIR/../../"
-tmpfile="$DIR/.cppcheck_errors.tmp"
-errfile="$DIR/.cppcheck_errors"
+tmpfile="$DIR/cppcheck_and_clang_format.tmp"
+errfile="$DIR/cppcheck_and_clang_format.txt"
-mode=diff
-if echo "$PROJECT" | grep "release" >/dev/null 2>/dev/null; then
- mode=release
-fi
+mode=${1:-diff}
+threads=$( cat /proc/cpuinfo | grep processor | wc -l )
if [ "$mode" == diff ]; then
files=$( git diff --name-only HEAD~1 HEAD | egrep '^(src|tests|poc)' | egrep '\.(hpp|h|cpp)$' )
flags=""
else
files=src/
- flags="-j$THREADS -Isrc"
+ flags="-j$threads -Isrc"
fi
cat > .cppcheck_suppressions <<EOF
diff --git a/tools/apollo/macro_benchmark_summary b/tools/github/macro_benchmark_summary
similarity index 100%
rename from tools/apollo/macro_benchmark_summary
rename to tools/github/macro_benchmark_summary