diff --git a/.github/workflows/diff.yaml b/.github/workflows/diff.yaml new file mode 100644 index 000000000..cac272151 --- /dev/null +++ b/.github/workflows/diff.yaml @@ -0,0 +1,284 @@ +name: Diff + +on: [push] + +jobs: + community_build: + name: "Community build" + runs-on: [self-hosted, Linux, X64, Debian10] + env: + THREADS: 24 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Build community binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + ./init + + # Build community binaries. + cd build + cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF .. + make -j$THREADS + + - name: Run unit tests + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run unit tests. + cd build + ctest -R memgraph__unit --output-on-failure + + - name: Run stress test + run: | + cd tests/stress + ./continuous_integration + + coverage_build: + name: "Coverage build" + runs-on: [self-hosted, Linux, X64, Debian10] + env: + THREADS: 24 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Build coverage binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + ./init + + # Build coverage binaries. + cd build + cmake -DTEST_COVERAGE=ON .. + make -j$THREADS memgraph__unit + + - name: Run unit tests + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run unit tests. + cd build + ctest -R memgraph__unit --output-on-failure + + - name: Compute code coverage + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Compute code coverage. + cd tools/github + ./coverage_convert + + # Package code coverage. + cd generated + tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu + + - name: Save code coverage + uses: actions/upload-artifact@v2 + with: + name: "Code coverage" + path: tools/github/generated/code_coverage.tar.gz + + debug_build: + name: "Debug build" + runs-on: [self-hosted, Linux, X64, Debian10] + env: + THREADS: 24 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Build debug binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + ./init + + # Build debug binaries. + cd build + cmake .. + make -j$THREADS + + - name: Run leftover CTest tests + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run leftover CTest tests (all except unit and benchmark tests). + cd build + ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure + + - name: Run drivers tests + run: | + ./tests/drivers/run.sh + + - name: Run integration tests + run: | + cd tests/integration + for name in *; do + if [ ! -d $name ]; then continue; fi + pushd $name >/dev/null + echo "Running: $name" + if [ -x prepare.sh ]; then + ./prepare.sh + fi + if [ -x runner.py ]; then + ./runner.py + elif [ -x runner.sh ]; then + ./runner.sh + fi + echo + popd >/dev/null + done + + - name: Run cppcheck and clang-format + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run cppcheck and clang-format. + cd tools/github + ./cppcheck_and_clang_format diff + + - name: Save cppcheck and clang-format errors + uses: actions/upload-artifact@v2 + with: + name: "Code coverage" + path: tools/github/cppcheck_and_clang_format.txt + + release_build: + name: "Release build" + runs-on: [self-hosted, Linux, X64, Debian10] + env: + THREADS: 24 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Set up parent + run: | + # Remove parent folder (if it exists). + cd .. + if [ -d parent ]; then + rm -rf parent + fi + + # Copy untouched repository to parent folder. + cp -r memgraph parent + + # Checkout previous commit + cd parent + git checkout HEAD~1 + + - name: Build release binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + ./init + + # Build release binaries. + cd build + cmake -DCMAKE_BUILD_TYPE=release .. + make -j$THREADS + + - name: Build parent binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + cd ../parent + ./init + + # Build parent binaries. + cd build + cmake -DCMAKE_BUILD_TYPE=release .. + make -j$THREADS memgraph memgraph__macro_benchmark + + - name: Run macro benchmark tests + run: | + cd tests/macro_benchmark + ./harness QuerySuite MemgraphRunner \ + --groups aggregation 1000_create unwind_create dense_expand match \ + --no-strict + + - name: Run parent macro benchmark tests + run: | + cd ../parent/tests/macro_benchmark + ./harness QuerySuite MemgraphRunner \ + --groups aggregation 1000_create unwind_create dense_expand match \ + --no-strict + + - name: Compute macro benchmark summary + run: | + ./tools/github/macro_benchmark_summary \ + --current tests/macro_benchmark/.harness_summary \ + --previous ../parent/tests/macro_benchmark/.harness_summary \ + --output macro_benchmark_summary.txt + + - name: Save macro benchmark summary + uses: actions/upload-artifact@v2 + with: + name: "Macro benchmark summary" + path: macro_benchmark_summary.txt + + - name: Run quality assurance tests + run: | + cd tests/qa + ./continuous_integration + + - name: Save quality assurance status + uses: actions/upload-artifact@v2 + with: + name: "Quality assurance status" + path: tests/qa/quality_assurance_status.txt + + - name: Run stress test (plain) + run: | + cd tests/stress + ./continuous_integration + + - name: Run stress test (SSL) + run: | + cd tests/stress + ./continuous_integration --use-ssl + + - name: Run durability test + run: | + cd tests/stress + source ve3/bin/activate + python3 durability --num-steps 5 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..db4da80a0 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,307 @@ +name: Release + +on: + workflow_dispatch: + schedule: + - cron: "0 1 * * *" + +jobs: + community_build: + name: "Community build" + runs-on: [self-hosted, Linux, X64, Debian10] + env: + THREADS: 24 + timeout-minutes: 960 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Build community binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + ./init + + # Build community binaries. + cd build + cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF .. + make -j$THREADS + + - name: Create community DEB package + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Create community DEB package. + cd build + mkdir output && cd output + cpack -G DEB --config ../CPackConfig.cmake + + - name: Save community DEB package + uses: actions/upload-artifact@v2 + with: + name: "Community DEB package" + path: build/output/memgraph*.deb + + - name: Run unit tests + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run unit tests. + cd build + ctest -R memgraph__unit --output-on-failure + + - name: Run stress test (plain) + run: | + cd tests/stress + ./continuous_integration + + - name: Run stress test (large) + run: | + cd tests/stress + ./continuous_integration --large-dataset + + coverage_build: + name: "Coverage build" + runs-on: [self-hosted, Linux, X64, Debian10] + env: + THREADS: 24 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Build coverage binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + ./init + + # Build coverage binaries. + cd build + cmake -DTEST_COVERAGE=ON .. + make -j$THREADS memgraph__unit + + - name: Run unit tests + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run unit tests. + cd build + ctest -R memgraph__unit --output-on-failure + + - name: Compute code coverage + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Compute code coverage. + cd tools/github + ./coverage_convert + + # Package code coverage. + cd generated + tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu + + - name: Save code coverage + uses: actions/upload-artifact@v2 + with: + name: "Code coverage" + path: tools/github/generated/code_coverage.tar.gz + + debug_build: + name: "Debug build" + runs-on: [self-hosted, Linux, X64, Debian10] + env: + THREADS: 24 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Build debug binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + ./init + + # Build debug binaries. + cd build + cmake .. + make -j$THREADS + + - name: Run leftover CTest tests + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run leftover CTest tests (all except unit and benchmark tests). + cd build + ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure + + - name: Run drivers tests + run: | + ./tests/drivers/run.sh + + - name: Run integration tests + run: | + cd tests/integration + for name in *; do + if [ ! -d $name ]; then continue; fi + pushd $name >/dev/null + echo "Running: $name" + if [ -x prepare.sh ]; then + ./prepare.sh + fi + if [ -x runner.py ]; then + ./runner.py + elif [ -x runner.sh ]; then + ./runner.sh + fi + echo + popd >/dev/null + done + + - name: Run cppcheck and clang-format + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run cppcheck and clang-format. + cd tools/github + ./cppcheck_and_clang_format diff + + - name: Save cppcheck and clang-format errors + uses: actions/upload-artifact@v2 + with: + name: "Code coverage" + path: tools/github/cppcheck_and_clang_format.txt + + release_build: + name: "Release build" + runs-on: [self-hosted, Linux, X64, Debian10] + env: + THREADS: 24 + timeout-minutes: 960 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Build release binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + ./init + + # Build release binaries. + cd build + cmake -DCMAKE_BUILD_TYPE=release .. + make -j$THREADS + + - name: Create enterprise DEB package + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Create enterprise DEB package. + cd build + mkdir output && cd output + cpack -G DEB --config ../CPackConfig.cmake + + - name: Save enterprise DEB package + uses: actions/upload-artifact@v2 + with: + name: "Enterprise DEB package" + path: build/output/memgraph*.deb + + - name: Run micro benchmark tests + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run micro benchmark tests. + cd build + # The `eval` benchmark needs a large stack limit. + ulimit -s 262144 + ctest -R memgraph__benchmark -V + + - name: Run macro benchmark tests + run: | + cd tests/macro_benchmark + ./harness QuerySuite MemgraphRunner \ + --groups aggregation 1000_create unwind_create dense_expand match \ + --no-strict + + - name: Run parallel macro benchmark tests + run: | + cd tests/macro_benchmark + ./harness QueryParallelSuite MemgraphRunner \ + --groups aggregation_parallel create_parallel bfs_parallel \ + --num-database-workers 9 --num-clients-workers 30 \ + --no-strict + + - name: Run quality assurance tests + run: | + cd tests/qa + ./continuous_integration + + - name: Save quality assurance status + uses: actions/upload-artifact@v2 + with: + name: "Quality assurance status" + path: tests/qa/quality_assurance_status.txt + + - name: Run stress test (plain) + run: | + cd tests/stress + ./continuous_integration + + - name: Run stress test (SSL) + run: | + cd tests/stress + ./continuous_integration --use-ssl + + - name: Run stress test (large) + run: | + cd tests/stress + ./continuous_integration --large-dataset + + - name: Run durability test (plain) + run: | + cd tests/stress + source ve3/bin/activate + python3 durability --num-steps 5 + + - name: Run durability test (large) + run: | + cd tests/stress + source ve3/bin/activate + python3 durability --num-steps 20 diff --git a/.github/workflows/release_centos.yaml b/.github/workflows/release_centos.yaml new file mode 100644 index 000000000..ae5009d6a --- /dev/null +++ b/.github/workflows/release_centos.yaml @@ -0,0 +1,306 @@ +name: Release CentOS + +on: [workflow_dispatch] + +jobs: + community_build: + name: "Community build" + runs-on: [self-hosted, Linux, X64, CentOS7] + env: + THREADS: 24 + timeout-minutes: 960 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Build community binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + ./init + + # Build community binaries. + cd build + cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF .. + make -j$THREADS + + - name: Create community RPM package + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Create community RPM package. + cd build + mkdir output && cd output + cpack -G RPM --config ../CPackConfig.cmake + rpmlint memgraph*.rpm + + - name: Save community RPM package + uses: actions/upload-artifact@v2 + with: + name: "Community RPM package" + path: build/output/memgraph*.rpm + + - name: Run unit tests + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run unit tests. + cd build + ctest -R memgraph__unit --output-on-failure + + - name: Run stress test (plain) + run: | + cd tests/stress + ./continuous_integration + + - name: Run stress test (large) + run: | + cd tests/stress + ./continuous_integration --large-dataset + + coverage_build: + name: "Coverage build" + runs-on: [self-hosted, Linux, X64, CentOS7] + env: + THREADS: 24 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Build coverage binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + ./init + + # Build coverage binaries. + cd build + cmake -DTEST_COVERAGE=ON .. + make -j$THREADS memgraph__unit + + - name: Run unit tests + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run unit tests. + cd build + ctest -R memgraph__unit --output-on-failure + + - name: Compute code coverage + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Compute code coverage. + cd tools/github + ./coverage_convert + + # Package code coverage. + cd generated + tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu + + - name: Save code coverage + uses: actions/upload-artifact@v2 + with: + name: "Code coverage" + path: tools/github/generated/code_coverage.tar.gz + + debug_build: + name: "Debug build" + runs-on: [self-hosted, Linux, X64, CentOS7] + env: + THREADS: 24 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Build debug binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + ./init + + # Build debug binaries. + cd build + cmake .. + make -j$THREADS + + - name: Run leftover CTest tests + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run leftover CTest tests (all except unit and benchmark tests). + cd build + ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure + + - name: Run drivers tests + run: | + ./tests/drivers/run.sh + + - name: Run integration tests + run: | + cd tests/integration + for name in *; do + if [ ! -d $name ]; then continue; fi + pushd $name >/dev/null + echo "Running: $name" + if [ -x prepare.sh ]; then + ./prepare.sh + fi + if [ -x runner.py ]; then + ./runner.py + elif [ -x runner.sh ]; then + ./runner.sh + fi + echo + popd >/dev/null + done + + - name: Run cppcheck and clang-format + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run cppcheck and clang-format. + cd tools/github + ./cppcheck_and_clang_format diff + + - name: Save cppcheck and clang-format errors + uses: actions/upload-artifact@v2 + with: + name: "Code coverage" + path: tools/github/cppcheck_and_clang_format.txt + + release_build: + name: "Release build" + runs-on: [self-hosted, Linux, X64, CentOS7] + env: + THREADS: 24 + timeout-minutes: 960 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Build release binaries + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Initialize dependencies. + ./init + + # Build release binaries. + cd build + cmake -DCMAKE_BUILD_TYPE=release .. + make -j$THREADS + + - name: Create enterprise RPM package + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Create enterprise RPM package. + cd build + mkdir output && cd output + cpack -G RPM --config ../CPackConfig.cmake + rpmlint memgraph*.rpm + + - name: Save enterprise RPM package + uses: actions/upload-artifact@v2 + with: + name: "Enterprise RPM package" + path: build/output/memgraph*.rpm + + - name: Run micro benchmark tests + run: | + # Activate toolchain. + source /opt/toolchain-v1/activate + + # Run micro benchmark tests. + cd build + # The `eval` benchmark needs a large stack limit. + ulimit -s 262144 + ctest -R memgraph__benchmark -V + + - name: Run macro benchmark tests + run: | + cd tests/macro_benchmark + ./harness QuerySuite MemgraphRunner \ + --groups aggregation 1000_create unwind_create dense_expand match \ + --no-strict + + - name: Run parallel macro benchmark tests + run: | + cd tests/macro_benchmark + ./harness QueryParallelSuite MemgraphRunner \ + --groups aggregation_parallel create_parallel bfs_parallel \ + --num-database-workers 9 --num-clients-workers 30 \ + --no-strict + + - name: Run quality assurance tests + run: | + cd tests/qa + ./continuous_integration + + - name: Save quality assurance status + uses: actions/upload-artifact@v2 + with: + name: "Quality assurance status" + path: tests/qa/quality_assurance_status.txt + + - name: Run stress test (plain) + run: | + cd tests/stress + ./continuous_integration + + - name: Run stress test (SSL) + run: | + cd tests/stress + ./continuous_integration --use-ssl + + - name: Run stress test (large) + run: | + cd tests/stress + ./continuous_integration --large-dataset + + - name: Run durability test (plain) + run: | + cd tests/stress + source ve3/bin/activate + python3 durability --num-steps 5 + + - name: Run durability test (large) + run: | + cd tests/stress + source ve3/bin/activate + python3 durability --num-steps 20 diff --git a/init b/init index b6f4033f2..795874fdd 100755 --- a/init +++ b/init @@ -1,22 +1,44 @@ #!/bin/bash -e -required_pkgs=(git # source code control - make pkg-config # build system - curl wget # for downloading libs - uuid-dev default-jre-headless # required by antlr - libreadline-dev # for memgraph console - libssl-dev - libseccomp-dev - python3 python-virtualenv python3-pip # for qa, macro_benchmark and stress tests - python3-yaml # for the configuration generator - libcurl4-openssl-dev # mg-requests - sbcl # for custom Lisp C++ preprocessing - ) +required_pkgs_apt=(git # source code control + make pkg-config # build system + curl wget # for downloading libs + uuid-dev default-jre-headless # required by antlr + libreadline-dev # for memgraph console + libpython3-dev python3-dev # for query modules + libssl-dev + libseccomp-dev + python3 python-virtualenv python3-pip # for qa, macro_benchmark and stress tests + python3-yaml # for the configuration generator + libcurl4-openssl-dev # mg-requests + sbcl # for custom Lisp C++ preprocessing + ) -optional_pkgs=(doxygen graphviz # source documentation generators - php-cli # for user technical documentation generators - mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests - ) +optional_pkgs_apt=(doxygen graphviz # source documentation generators + php-cli # for user technical documentation generators + mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests + ) + + +required_pkgs_yum=(# NOTE: git is too old on CentOS, install it manually from IUS + make pkgconfig # build system + curl wget # for downloading libs + libuuid-devel java-1.8.0-openjdk # required by antlr + readline-devel # for memgraph console + python3-devel # for query modules + openssl-devel + libseccomp-devel + python3 python-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests + # NOTE: python3-yaml doesn't exist on CentOS, install it manually using `pip3 install PyYAML` + libcurl-devel # mg-requests + sbcl # for custom Lisp C++ preprocessing + rpm-build rpmlint # for RPM package building + ) + +optional_pkgs_yum=(doxygen graphviz # source documentation generators + php-cli # for user technical documentation generators + mono-complete nodejs zip unzip java-1.8.0-openjdk-devel # for driver tests + ) use_sudo=0 @@ -76,29 +98,55 @@ echo "Started installing dependencies for Memgraph" required_missing=0 -# install all dependencies on debian based operating systems -for pkg in ${required_pkgs[@]}; do - if dpkg -s $pkg 2>/dev/null >/dev/null; then - echo "Found $pkg" - elif (( $use_sudo )); then - echo "Installing $pkg" - if [[ ! `sudo apt-get -y install $pkg` ]]; then - echo "Didn't install $pkg [required]" +# install all dependencies on supported operating systems +if command -v apt-get >/dev/null; then + for pkg in ${required_pkgs_apt[@]}; do + if dpkg -s $pkg 2>/dev/null >/dev/null; then + echo "Found $pkg" + elif (( $use_sudo )); then + echo "Installing $pkg" + if [[ ! `sudo apt-get -y install $pkg` ]]; then + echo "Didn't install $pkg [required]" + required_missing=1 + fi + else + echo "Missing $pkg [required]" required_missing=1 fi - else - echo "Missing $pkg [required]" - required_missing=1 - fi -done - -for pkg in ${optional_pkgs[@]}; do - if dpkg -s $pkg 2>/dev/null >/dev/null; then - echo "Found $pkg [optional]" - else - echo "Missing $pkg [optional]" - fi -done + done + for pkg in ${optional_pkgs_apt[@]}; do + if dpkg -s $pkg 2>/dev/null >/dev/null; then + echo "Found $pkg [optional]" + else + echo "Missing $pkg [optional]" + fi + done +elif command -v yum >/dev/null; then + for pkg in ${required_pkgs_yum[@]}; do + if yum list installed $pkg 2>/dev/null >/dev/null; then + echo "Found $pkg" + elif (( $use_sudo )); then + echo "Installing $pkg" + if [[ ! `sudo yum install -y $pkg` ]]; then + echo "Didn't install $pkg [required]" + required_missing=1 + fi + else + echo "Missing $pkg [required]" + required_missing=1 + fi + done + for pkg in ${optional_pkgs_yum[@]}; do + if yum list installed $pkg 2>/dev/null >/dev/null; then + echo "Found $pkg [optional]" + else + echo "Missing $pkg [optional]" + fi + done +else + echo "Unsupported distribution!" + exit 1 +fi if (( $required_missing )); then echo "Missing required packages. EXITING!" diff --git a/tests/drivers/javascript/run.sh b/tests/drivers/javascript/run.sh index f003a5175..4528f4e4f 100755 --- a/tests/drivers/javascript/run.sh +++ b/tests/drivers/javascript/run.sh @@ -3,7 +3,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$DIR" -if ! which nodejs >/dev/null; then +if ! which node >/dev/null; then echo "Please install nodejs!" exit 1 fi @@ -15,5 +15,5 @@ if [ ! -d node_modules ]; then rm driver.tar.gz || exit 1 fi -nodejs basic.js -nodejs max_query_length.js +node basic.js +node max_query_length.js diff --git a/tests/drivers/run.sh b/tests/drivers/run.sh index c98799b0a..0243e7e2b 100755 --- a/tests/drivers/run.sh +++ b/tests/drivers/run.sh @@ -31,7 +31,8 @@ fi $binary_dir/memgraph \ --data-directory=$tmpdir \ --query-execution-timeout-sec=5 \ - --bolt-session-inactivity-timeout=10 & + --bolt-session-inactivity-timeout=10 \ + --min-log-level 1 & pid=$! wait_for_server 7687 @@ -53,7 +54,7 @@ done # Stop memgraph. kill $pid -wait -n +wait $pid code_mg=$? # Temporary directory cleanup. diff --git a/tests/integration/transactions/runner.sh b/tests/integration/transactions/runner.sh index 10195fb59..432907630 100755 --- a/tests/integration/transactions/runner.sh +++ b/tests/integration/transactions/runner.sh @@ -23,7 +23,7 @@ code=$? # Shutdown the memgraph process. kill $pid -wait -n +wait $pid code_mg=$? # Check memgraph exit code. diff --git a/tests/qa/.gitignore b/tests/qa/.gitignore index 388af07eb..26a36bb39 100644 --- a/tests/qa/.gitignore +++ b/tests/qa/.gitignore @@ -4,4 +4,4 @@ *~ *.pyc ve3/ -.quality_assurance_status +quality_assurance_status.txt diff --git a/tests/qa/continuous_integration b/tests/qa/continuous_integration index b88b48da5..29b12cba7 100755 --- a/tests/qa/continuous_integration +++ b/tests/qa/continuous_integration @@ -27,7 +27,7 @@ import yaml SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) TESTS_DIR = os.path.join(SCRIPT_DIR, "tests") BASE_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "..")) -BUILD_DIR = os.path.join(BASE_DIR, "build_release") +BUILD_DIR = os.path.join(BASE_DIR, "build") def wait_for_server(port, delay=0.01): @@ -131,7 +131,7 @@ def main(): # Load tests from config file with open(os.path.join(TESTS_DIR, "config.yaml")) as f: - suites = yaml.load(f) + suites = yaml.safe_load(f) # venv used to run the qa engine venv_python = os.path.join(SCRIPT_DIR, "ve3", "bin", "python3") @@ -184,7 +184,7 @@ def main(): qa_status_message = generate_remarkup(status_data) # Create the report file - qa_status_path = os.path.join(SCRIPT_DIR, ".quality_assurance_status") + qa_status_path = os.path.join(SCRIPT_DIR, "quality_assurance_status.txt") with open(qa_status_path, "w") as f: f.write(qa_status_message) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 4bb88f5e0..db97cb2d5 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -13,7 +13,11 @@ function(add_unit_test test_cpp) set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name}) target_link_libraries(${target_name} gtest gmock gtest_main Threads::Threads) # register test - add_test(${target_name} ${exec_name}) + if(TEST_COVERAGE) + add_test(${target_name} env LLVM_PROFILE_FILE=${exec_name}.profraw ./${exec_name}) + else() + add_test(${target_name} ${exec_name}) + endif() # add to memgraph__unit target add_dependencies(memgraph__unit ${target_name}) endfunction(add_unit_test) diff --git a/tools/github/.gitignore b/tools/github/.gitignore new file mode 100644 index 000000000..8c743badd --- /dev/null +++ b/tools/github/.gitignore @@ -0,0 +1,2 @@ +cppcheck_and_clang_format.txt +generated/* diff --git a/tools/apollo/coverage_convert b/tools/github/coverage_convert similarity index 57% rename from tools/apollo/coverage_convert rename to tools/github/coverage_convert index 15280f747..232ef697d 100755 --- a/tools/apollo/coverage_convert +++ b/tools/github/coverage_convert @@ -2,13 +2,15 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" project_dir="$( dirname "$( dirname "$script_dir" )" )" +source_dir="$project_dir/src" +unit_tests_dir="$project_dir/build/tests/unit" generated_dir="$script_dir/generated" html_dir="$generated_dir/html" data_file="$generated_dir/default.profdata" json_file="$generated_dir/report.json" coverage_file="$generated_dir/coverage.json" -summary_file="$generated_dir/.coverage_summary" +summary_file="$generated_dir/summary.rmu" # cleanup output directory if [ -d "$generated_dir" ]; then @@ -17,22 +19,14 @@ fi mkdir "$generated_dir" # merge raw coverage info -raw_files="$( find "$HOME" -name "*.profraw" | tr '\n' ' ' )" +raw_files="$( find "$unit_tests_dir" -name "*.profraw" | sort | tr '\n' ' ' )" llvm-profdata merge -sparse $raw_files -o "$data_file" # create list of binaries -cnt=0 -obj_files="" -for prof_file in $raw_files; do - if [ $cnt -gt 0 ]; then - obj_files+=" -object " - fi - obj_files+="$( find "$( dirname "$prof_file" )" -executable -type f | head -n 1 )" - cnt=$((cnt + 1)) -done +obj_files="$( echo "$raw_files" | sed "s/\.profraw//g" | sed -r 's/ +$//g' | sed "s/ / -object /g" )" # create list of source files -src_files=$( find "$HOME" \( -name '*.cpp' -o -name '*.hpp' \) -print | sort | tr '\n' ' ' ) +src_files=$( find "$source_dir" \( -name '*.cpp' -o -name '*.hpp' \) -print | sort | tr '\n' ' ' ) # generate html output llvm-cov show $obj_files \ @@ -43,14 +37,6 @@ llvm-cov show $obj_files \ -Xdemangler c++filt -Xdemangler -n \ $src_files -# fix names in html output -coverage_dir="$html_dir/coverage" -mv $coverage_dir/workspace/memgraph/* $html_dir/coverage/ -rm -r $coverage_dir/workspace -find $coverage_dir -name "*.html" -exec sed -i 's@/workspace/memgraph/@@g' {} \; -find $coverage_dir -name "*.html" -exec sed -i 's@../../style.css@style.css@g' {} \; -sed -i 's@/workspace/memgraph@@g' $html_dir/index.html - # generate json output llvm-cov export $obj_files \ -instr-profile "$data_file" \ diff --git a/tools/apollo/coverage_parse_export b/tools/github/coverage_parse_export similarity index 100% rename from tools/apollo/coverage_parse_export rename to tools/github/coverage_parse_export diff --git a/tools/apollo/cppcheck b/tools/github/cppcheck_and_clang_format similarity index 91% rename from tools/apollo/cppcheck rename to tools/github/cppcheck_and_clang_format index 188af1d4f..044fb270b 100755 --- a/tools/apollo/cppcheck +++ b/tools/github/cppcheck_and_clang_format @@ -3,20 +3,18 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$DIR/../../" -tmpfile="$DIR/.cppcheck_errors.tmp" -errfile="$DIR/.cppcheck_errors" +tmpfile="$DIR/cppcheck_and_clang_format.tmp" +errfile="$DIR/cppcheck_and_clang_format.txt" -mode=diff -if echo "$PROJECT" | grep "release" >/dev/null 2>/dev/null; then - mode=release -fi +mode=${1:-diff} +threads=$( cat /proc/cpuinfo | grep processor | wc -l ) if [ "$mode" == diff ]; then files=$( git diff --name-only HEAD~1 HEAD | egrep '^(src|tests|poc)' | egrep '\.(hpp|h|cpp)$' ) flags="" else files=src/ - flags="-j$THREADS -Isrc" + flags="-j$threads -Isrc" fi cat > .cppcheck_suppressions <