From 96cb02ee96255c5374f8d4b94b44f6dba9acf175 Mon Sep 17 00:00:00 2001
From: Deda <marko.barisic@memgraph.io>
Date: Fri, 22 Mar 2024 17:33:00 +0100
Subject: [PATCH] Minor final changes
---
.github/workflows/diff.yaml | 14 +-
.github/workflows/release_ubuntu2004.yaml | 417 ----------------------
.github/workflows/stress_test_large.yaml | 2 +-
release/package/mgbuild.sh | 9 +-
4 files changed, 9 insertions(+), 433 deletions(-)
delete mode 100644 .github/workflows/release_ubuntu2004.yaml
diff --git a/.github/workflows/diff.yaml b/.github/workflows/diff.yaml
index 91229893c..8b8ee607f 100644
--- a/.github/workflows/diff.yaml
+++ b/.github/workflows/diff.yaml
@@ -8,13 +8,13 @@ on:
branches:
- master
workflow_dispatch:
- # pull_request:
- # paths-ignore:
- # - "docs/**"
- # - "**/*.md"
- # - ".clang-format"
- # - "CODEOWNERS"
- # - "licenses/*"
+ pull_request:
+ paths-ignore:
+ - "docs/**"
+ - "**/*.md"
+ - ".clang-format"
+ - "CODEOWNERS"
+ - "licenses/*"
jobs:
community_build:
diff --git a/.github/workflows/release_ubuntu2004.yaml b/.github/workflows/release_ubuntu2004.yaml
deleted file mode 100644
index be6099128..000000000
--- a/.github/workflows/release_ubuntu2004.yaml
+++ /dev/null
@@ -1,417 +0,0 @@
-name: Release Ubuntu 20.04
-
-on:
- workflow_call:
- inputs:
- build_type:
- type: string
- description: "Memgraph Build type. Default value is Release."
- default: 'Release'
- workflow_dispatch:
- inputs:
- build_type:
- type: choice
- description: "Memgraph Build type. Default value is Release."
- default: 'Release'
- options:
- - Release
- - RelWithDebInfo
-
-env:
- OS: "Ubuntu 20.04"
- THREADS: 24
- MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
- MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
- BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
-
-jobs:
- community_build:
- name: "Community build"
- runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
- timeout-minutes: 60
-
- steps:
- - name: Set up repository
- uses: actions/checkout@v4
- with:
- # Number of commits to fetch. `0` indicates all history for all
- # branches and tags. (default: 1)
- fetch-depth: 0
-
- - name: Build community binaries
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Initialize dependencies.
- ./init
-
- # Build community binaries.
- cd build
- cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DMG_ENTERPRISE=OFF ..
- make -j$THREADS
-
- - name: Run unit tests
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Run unit tests.
- cd build
- ctest -R memgraph__unit --output-on-failure
-
- coverage_build:
- name: "Coverage build"
- runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
- timeout-minutes: 60
-
- steps:
- - name: Set up repository
- uses: actions/checkout@v4
- with:
- # Number of commits to fetch. `0` indicates all history for all
- # branches and tags. (default: 1)
- fetch-depth: 0
-
- - name: Build coverage binaries
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Initialize dependencies.
- ./init
-
- # Build coverage binaries.
- cd build
- cmake -DTEST_COVERAGE=ON ..
- make -j$THREADS memgraph__unit
-
- - name: Run unit tests
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Run unit tests.
- cd build
- ctest -R memgraph__unit --output-on-failure
-
- - name: Compute code coverage
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Compute code coverage.
- cd tools/github
- ./coverage_convert
-
- # Package code coverage.
- cd generated
- tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
-
- - name: Save code coverage
- uses: actions/upload-artifact@v4
- with:
- name: "Code coverage(Coverage build)-${{ env.OS }}"
- path: tools/github/generated/code_coverage.tar.gz
-
- debug_build:
- name: "Debug build"
- runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
- timeout-minutes: 60
-
- steps:
- - name: Set up repository
- uses: actions/checkout@v4
- with:
- # Number of commits to fetch. `0` indicates all history for all
- # branches and tags. (default: 1)
- fetch-depth: 0
-
- - name: Build debug binaries
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Initialize dependencies.
- ./init
-
- # Build debug binaries.
- cd build
- cmake ..
- make -j$THREADS
-
- - name: Run leftover CTest tests
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Run leftover CTest tests (all except unit and benchmark tests).
- cd build
- ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
-
- - name: Run drivers tests
- run: |
- ./tests/drivers/run.sh
-
- - name: Run cppcheck and clang-format
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Run cppcheck and clang-format.
- cd tools/github
- ./cppcheck_and_clang_format diff
-
- - name: Save cppcheck and clang-format errors
- uses: actions/upload-artifact@v4
- with:
- name: "Code coverage(Debug build)-${{ env.OS }}"
- path: tools/github/cppcheck_and_clang_format.txt
-
- debug_integration_test:
- name: "Debug integration tests"
- runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
- timeout-minutes: 60
-
- steps:
- - name: Set up repository
- uses: actions/checkout@v4
- with:
- # Number of commits to fetch. `0` indicates all history for all
- # branches and tags. (default: 1)
- fetch-depth: 0
-
- - name: Build debug binaries
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Initialize dependencies.
- ./init
-
- # Build debug binaries.
- cd build
- cmake ..
- make -j$THREADS
-
- - name: Run integration tests
- run: |
- tests/integration/run.sh
-
- release_build:
- name: "Release build"
- runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
- timeout-minutes: 60
-
- steps:
- - name: Set up repository
- uses: actions/checkout@v4
- with:
- # Number of commits to fetch. `0` indicates all history for all
- # branches and tags. (default: 1)
- fetch-depth: 0
-
- - name: Build release binaries
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Initialize dependencies.
- ./init
-
- # Build release binaries.
- cd build
- cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
- make -j$THREADS
-
- - name: Create enterprise DEB package
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- cd build
-
- # create mgconsole
- # we use the -B to force the build
- make -j$THREADS -B mgconsole
-
- # Create enterprise DEB package.
- mkdir output && cd output
- cpack -G DEB --config ../CPackConfig.cmake
-
- - name: Save enterprise DEB package
- uses: actions/upload-artifact@v4
- with:
- name: "Enterprise DEB package-${{ env.OS }}"
- path: build/output/memgraph*.deb
-
- - name: Run GQL Behave tests
- run: |
- cd tests
- ./setup.sh /opt/toolchain-v4/activate
- cd gql_behave
- ./continuous_integration
-
- - name: Save quality assurance status
- uses: actions/upload-artifact@v4
- with:
- name: "GQL Behave Status-${{ env.OS }}"
- path: |
- tests/gql_behave/gql_behave_status.csv
- tests/gql_behave/gql_behave_status.html
-
- - name: Run unit tests
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Run unit tests.
- cd build
- ctest -R memgraph__unit --output-on-failure
-
- release_benchmark_tests:
- name: "Release Benchmark Tests"
- runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
- timeout-minutes: 60
-
- steps:
- - name: Set up repository
- uses: actions/checkout@v4
- with:
- # Number of commits to fetch. `0` indicates all history for all
- # branches and tags. (default: 1)
- fetch-depth: 0
-
- - name: Build release binaries
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
- # Initialize dependencies.
- ./init
-
- # Build release binaries
- cd build
- cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
- make -j$THREADS
-
- - name: Run micro benchmark tests
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Run micro benchmark tests.
- cd build
- # The `eval` benchmark needs a large stack limit.
- ulimit -s 262144
- ctest -R memgraph__benchmark -V
-
- - name: Run macro benchmark tests
- run: |
- cd tests/macro_benchmark
- ./harness QuerySuite MemgraphRunner \
- --groups aggregation 1000_create unwind_create dense_expand match \
- --no-strict
-
- - name: Run parallel macro benchmark tests
- run: |
- cd tests/macro_benchmark
- ./harness QueryParallelSuite MemgraphRunner \
- --groups aggregation_parallel create_parallel bfs_parallel \
- --num-database-workers 9 --num-clients-workers 30 \
- --no-strict
-
- release_e2e_test:
- name: "Release End-to-end Test"
- runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
- timeout-minutes: 60
-
- steps:
- - name: Set up repository
- uses: actions/checkout@v4
- with:
- # Number of commits to fetch. `0` indicates all history for all
- # branches and tags. (default: 1)
- fetch-depth: 0
-
- - name: Build release binaries
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
- # Initialize dependencies.
- ./init
-
- # Build release binaries
- cd build
- cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
- make -j$THREADS
-
- - name: Ensure Kafka and Pulsar are up
- run: |
- cd tests/e2e/streams/kafka
- docker-compose up -d
- cd ../pulsar
- docker-compose up -d
-
- - name: Run e2e tests
- run: |
- cd tests
- ./setup.sh /opt/toolchain-v4/activate
- source ve3/bin/activate_e2e
- cd e2e
- ./run.sh
-
- - name: Ensure Kafka and Pulsar are down
- if: always()
- run: |
- cd tests/e2e/streams/kafka
- docker-compose down
- cd ../pulsar
- docker-compose down
-
- release_durability_stress_tests:
- name: "Release durability and stress tests"
- runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
- timeout-minutes: 60
-
- steps:
- - name: Set up repository
- uses: actions/checkout@v4
- with:
- # Number of commits to fetch. `0` indicates all history for all
- # branches and tags. (default: 1)
- fetch-depth: 0
-
- - name: Build release binaries
- run: |
- # Activate toolchain.
- source /opt/toolchain-v4/activate
-
- # Initialize dependencies.
- ./init
-
- # Build release binaries.
- cd build
- cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
- make -j$THREADS
-
- - name: Run stress test (plain)
- run: |
- cd tests/stress
- ./continuous_integration
-
- - name: Run stress test (SSL)
- run: |
- cd tests/stress
- ./continuous_integration --use-ssl
-
- - name: Run durability test (plain)
- run: |
- cd tests/stress
- source ve3/bin/activate
- python3 durability --num-steps 5
-
- - name: Run durability test (large)
- run: |
- cd tests/stress
- source ve3/bin/activate
- python3 durability --num-steps 20
diff --git a/.github/workflows/stress_test_large.yaml b/.github/workflows/stress_test_large.yaml
index 01b6e531c..4737b258e 100644
--- a/.github/workflows/stress_test_large.yaml
+++ b/.github/workflows/stress_test_large.yaml
@@ -41,7 +41,7 @@ on:
- "v*.*-rc*"
schedule:
- cron: "0 22 * * *"
-# ${{ (github.event.inputs.target_os == 'debian-10' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v4' }}
+
jobs:
StressTestLargeIndividual:
if: ${{ !github.event.inputs.os || github.event.inputs.os != 'all' }}
diff --git a/release/package/mgbuild.sh b/release/package/mgbuild.sh
index bb58b4428..e68e56edd 100755
--- a/release/package/mgbuild.sh
+++ b/release/package/mgbuild.sh
@@ -488,14 +488,6 @@ test_memgraph() {
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && export THREADS=$threads && $ACTIVATE_TOOLCHAIN && cd $MGBUILD_ROOT_DIR/tests/code_analysis && $SETUP_PASSED_ARGS "'&& ./clang_tidy.sh $PASSED_ARGS'
;;
e2e)
- # local kafka_container="kafka_kafka_1"
- # local kafka_hostname="kafka"
- # local pulsar_container="pulsar_pulsar_1"
- # local pulsar_hostname="pulsar"
- # local setup_hostnames="export KAFKA_HOSTNAME=$kafka_hostname && PULSAR_HOSTNAME=$pulsar_hostname"
- # local build_container_network=$(docker inspect $build_container --format='{{ .HostConfig.NetworkMode }}')
- # docker network connect --alias $kafka_hostname $build_container_network $kafka_container > /dev/null 2>&1 || echo "Kafka container already inside correct network or something went wrong ..."
- # docker network connect --alias $pulsar_hostname $build_container_network $pulsar_container > /dev/null 2>&1 || echo "Kafka container already inside correct network or something went wrong ..."
docker exec -u mg $build_container bash -c "pip install --user networkx && pip3 install --user networkx"
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && $ACTIVATE_CARGO && cd $MGBUILD_ROOT_DIR/tests && $ACTIVATE_VENV && source ve3/bin/activate_e2e && cd $MGBUILD_ROOT_DIR/tests/e2e "'&& ./run.sh'
;;
@@ -506,6 +498,7 @@ test_memgraph() {
esac
}
+
##################################################
################### PARSE ARGS ###################
##################################################