diff --git a/.github/workflows/diff.yaml b/.github/workflows/diff.yaml index 49b7d4273..73052f23e 100644 --- a/.github/workflows/diff.yaml +++ b/.github/workflows/diff.yaml @@ -8,13 +8,13 @@ on: branches: - master workflow_dispatch: - pull_request: - paths-ignore: - - "docs/**" - - "**/*.md" - - ".clang-format" - - "CODEOWNERS" - - "licenses/*" + # pull_request: + # paths-ignore: + # - "docs/**" + # - "**/*.md" + # - ".clang-format" + # - "CODEOWNERS" + # - "licenses/*" jobs: community_build: diff --git a/.github/workflows/release_build_test.yaml b/.github/workflows/release_build_test.yaml index cc4884758..4a3afa607 100644 --- a/.github/workflows/release_build_test.yaml +++ b/.github/workflows/release_build_test.yaml @@ -31,19 +31,24 @@ env: BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }} jobs: - Debian10: - uses: ./.github/workflows/release_debian10.yaml + Debian11: + uses: ./.github/workflows/reusable_release_tests.yaml with: + os: "debian-11" + toolchain: "v5" + arch: "amd" + runner_arch_label: "X64" build_type: ${{ github.event.inputs.build_type || 'Release' }} secrets: inherit Ubuntu20_04: + if: false uses: ./.github/workflows/release_ubuntu2004.yaml with: build_type: ${{ github.event.inputs.build_type || 'Release' }} secrets: inherit - PackageDebian10: + PackageDebian11: if: github.ref_type == 'tag' needs: [Debian10] runs-on: [self-hosted, DockerMgBuild, X64] @@ -71,102 +76,6 @@ jobs: name: debian-10 path: build/output/debian-10/memgraph*.deb - PackageUbuntu20_04: - if: github.ref_type == 'tag' - needs: [Ubuntu20_04] - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package ubuntu-22.04 $BUILD_TYPE - - name: Upload to S3 - uses: jakejarvis/s3-sync-action@v0.5.1 - env: - AWS_S3_BUCKET: "deps.memgraph.io" - AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: "eu-west-1" - SOURCE_DIR: "build/output" - DEST_DIR: "memgraph-unofficial/${{ github.ref_name }}/" - - name: "Upload package" - uses: actions/upload-artifact@v4 - with: - name: ubuntu-22.04 - path: build/output/ubuntu-22.04/memgraph*.deb - - PackageUbuntu20_04_ARM: - if: github.ref_type == 'tag' - needs: [Ubuntu20_04] - runs-on: [self-hosted, DockerMgBuild, ARM64] - # M1 Mac mini is sometimes slower - timeout-minutes: 150 - steps: - - name: "Set up repository" - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package ubuntu-22.04-arm $BUILD_TYPE - - name: "Upload package" - uses: actions/upload-artifact@v4 - with: - name: ubuntu-22.04-aarch64 - path: build/output/ubuntu-22.04-arm/memgraph*.deb - - PushToS3Ubuntu20_04_ARM: - if: github.ref_type == 'tag' - needs: [PackageUbuntu20_04_ARM] - runs-on: ubuntu-latest - steps: - - name: Download package - uses: actions/download-artifact@v4 - with: - name: ubuntu-22.04-aarch64 - path: build/output/release - - name: Upload to S3 - uses: jakejarvis/s3-sync-action@v0.5.1 - env: - AWS_S3_BUCKET: "deps.memgraph.io" - AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: "eu-west-1" - SOURCE_DIR: "build/output/release" - DEST_DIR: "memgraph-unofficial/${{ github.ref_name }}/" - - PackageDebian11: - if: github.ref_type == 'tag' - needs: [Debian10, Ubuntu20_04] - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package debian-11 $BUILD_TYPE - - name: Upload to S3 - uses: jakejarvis/s3-sync-action@v0.5.1 - env: - AWS_S3_BUCKET: "deps.memgraph.io" - AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: "eu-west-1" - SOURCE_DIR: "build/output" - DEST_DIR: "memgraph-unofficial/${{ github.ref_name }}/" - - name: "Upload package" - uses: actions/upload-artifact@v4 - with: - name: debian-11 - path: build/output/debian-11/memgraph*.deb - PackageDebian11_ARM: if: github.ref_type == 'tag' needs: [Debian10, Ubuntu20_04] diff --git a/.github/workflows/release_debian10.yaml b/.github/workflows/release_debian10.yaml deleted file mode 100644 index 188617a6e..000000000 --- a/.github/workflows/release_debian10.yaml +++ /dev/null @@ -1,463 +0,0 @@ -name: Release Debian 10 - -on: - workflow_call: - inputs: - build_type: - type: string - description: "Memgraph Build type. Default value is Release." - default: 'Release' - workflow_dispatch: - inputs: - build_type: - type: choice - description: "Memgraph Build type. Default value is Release." - default: 'Release' - options: - - Release - - RelWithDebInfo - -env: - OS: "Debian10" - THREADS: 24 - MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }} - MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }} - BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }} - -jobs: - community_build: - name: "Community build" - runs-on: [self-hosted, Linux, X64, Debian10] - timeout-minutes: 60 - - steps: - - name: Set up repository - uses: actions/checkout@v4 - with: - # Number of commits to fetch. `0` indicates all history for all - # branches and tags. (default: 1) - fetch-depth: 0 - - - name: Build community binaries - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Initialize dependencies. - ./init - - # Build community binaries. - cd build - cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DMG_ENTERPRISE=OFF .. - make -j$THREADS - - - name: Run unit tests - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Run unit tests. - cd build - ctest -R memgraph__unit --output-on-failure - - coverage_build: - name: "Coverage build" - runs-on: [self-hosted, Linux, X64, Debian10] - env: - THREADS: 24 - MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }} - MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }} - timeout-minutes: 60 - - steps: - - name: Set up repository - uses: actions/checkout@v4 - with: - # Number of commits to fetch. `0` indicates all history for all - # branches and tags. (default: 1) - fetch-depth: 0 - - - name: Build coverage binaries - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Initialize dependencies. - ./init - - # Build coverage binaries. - cd build - cmake -DTEST_COVERAGE=ON .. - make -j$THREADS memgraph__unit - - - name: Run unit tests - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Run unit tests. - cd build - ctest -R memgraph__unit --output-on-failure - - - name: Compute code coverage - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Compute code coverage. - cd tools/github - ./coverage_convert - - # Package code coverage. - cd generated - tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu - - - name: Save code coverage - uses: actions/upload-artifact@v4 - with: - name: "Code coverage(Coverage build)-${{ env.OS }}" - path: tools/github/generated/code_coverage.tar.gz - - debug_build: - name: "Debug build" - runs-on: [self-hosted, Linux, X64, Debian10] - timeout-minutes: 60 - - steps: - - name: Set up repository - uses: actions/checkout@v4 - with: - # Number of commits to fetch. `0` indicates all history for all - # branches and tags. (default: 1) - fetch-depth: 0 - - - name: Build debug binaries - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Initialize dependencies. - ./init - - # Build debug binaries. - cd build - cmake .. - make -j$THREADS - - - name: Run leftover CTest tests - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Run leftover CTest tests (all except unit and benchmark tests). - cd build - ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure - - - name: Run drivers tests - run: | - ./tests/drivers/run.sh - - - name: Run cppcheck and clang-format - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Run cppcheck and clang-format. - cd tools/github - ./cppcheck_and_clang_format diff - - - name: Save cppcheck and clang-format errors - uses: actions/upload-artifact@v4 - with: - name: "Code coverage(Debug build)-${{ env.OS }}" - path: tools/github/cppcheck_and_clang_format.txt - - debug_integration_test: - name: "Debug integration tests" - runs-on: [self-hosted, Linux, X64, Debian10] - timeout-minutes: 60 - - steps: - - name: Set up repository - uses: actions/checkout@v4 - with: - # Number of commits to fetch. `0` indicates all history for all - # branches and tags. (default: 1) - fetch-depth: 0 - - - name: Build debug binaries - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Initialize dependencies. - ./init - - # Build debug binaries. - cd build - cmake .. - make -j$THREADS - - - name: Run integration tests - run: | - tests/integration/run.sh - - release_build: - name: "Release build" - runs-on: [self-hosted, Linux, X64, Debian10] - timeout-minutes: 60 - - steps: - - name: Set up repository - uses: actions/checkout@v4 - with: - # Number of commits to fetch. `0` indicates all history for all - # branches and tags. (default: 1) - fetch-depth: 0 - - - name: Build release binaries - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Initialize dependencies. - ./init - - # Build release binaries. - cd build - cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. - make -j$THREADS - - - name: Create enterprise DEB package - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - cd build - - # create mgconsole - # we use the -B to force the build - make -j$THREADS -B mgconsole - - # Create enterprise DEB package. - mkdir output && cd output - cpack -G DEB --config ../CPackConfig.cmake - - - name: Save enterprise DEB package - uses: actions/upload-artifact@v4 - with: - name: "Enterprise DEB package-${{ env.OS}}" - path: build/output/memgraph*.deb - - - name: Run GQL Behave tests - run: | - cd tests - ./setup.sh /opt/toolchain-v4/activate - cd gql_behave - ./continuous_integration - - - name: Save quality assurance status - uses: actions/upload-artifact@v4 - with: - name: "GQL Behave Status-${{ env.OS }}" - path: | - tests/gql_behave/gql_behave_status.csv - tests/gql_behave/gql_behave_status.html - - - name: Run unit tests - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Run unit tests. - cd build - ctest -R memgraph__unit --output-on-failure - - release_benchmark_tests: - name: "Release Benchmark Tests" - runs-on: [self-hosted, Linux, X64, Debian10] - timeout-minutes: 60 - - steps: - - name: Set up repository - uses: actions/checkout@v4 - with: - # Number of commits to fetch. `0` indicates all history for all - # branches and tags. (default: 1) - fetch-depth: 0 - - - name: Build release binaries - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - # Initialize dependencies. - ./init - - # Build release binaries - cd build - cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. - make -j$THREADS - - - name: Run micro benchmark tests - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Run micro benchmark tests. - cd build - # The `eval` benchmark needs a large stack limit. - ulimit -s 262144 - ctest -R memgraph__benchmark -V - - - name: Run macro benchmark tests - run: | - cd tests/macro_benchmark - ./harness QuerySuite MemgraphRunner \ - --groups aggregation 1000_create unwind_create dense_expand match \ - --no-strict - - - name: Run parallel macro benchmark tests - run: | - cd tests/macro_benchmark - ./harness QueryParallelSuite MemgraphRunner \ - --groups aggregation_parallel create_parallel bfs_parallel \ - --num-database-workers 9 --num-clients-workers 30 \ - --no-strict - - release_e2e_test: - name: "Release End-to-end Test" - runs-on: [self-hosted, Linux, X64, Debian10] - timeout-minutes: 60 - - steps: - - name: Set up repository - uses: actions/checkout@v4 - with: - # Number of commits to fetch. `0` indicates all history for all - # branches and tags. (default: 1) - fetch-depth: 0 - - - name: Build release binaries - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - # Initialize dependencies. - ./init - - # Build release binaries - cd build - cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. - make -j$THREADS - - - name: Ensure Kafka and Pulsar are up - run: | - cd tests/e2e/streams/kafka - docker-compose up -d - cd ../pulsar - docker-compose up -d - - - name: Run e2e tests - run: | - cd tests - ./setup.sh /opt/toolchain-v4/activate - source ve3/bin/activate_e2e - cd e2e - ./run.sh - - - name: Ensure Kafka and Pulsar are down - if: always() - run: | - cd tests/e2e/streams/kafka - docker-compose down - cd ../pulsar - docker-compose down - - release_durability_stress_tests: - name: "Release durability and stress tests" - runs-on: [self-hosted, Linux, X64, Debian10] - timeout-minutes: 60 - - steps: - - name: Set up repository - uses: actions/checkout@v4 - with: - # Number of commits to fetch. `0` indicates all history for all - # branches and tags. (default: 1) - fetch-depth: 0 - - - name: Build release binaries - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Initialize dependencies. - ./init - - # Build release binaries. - cd build - cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. - make -j$THREADS - - - name: Run stress test (plain) - run: | - cd tests/stress - ./continuous_integration - - - name: Run stress test (SSL) - run: | - cd tests/stress - ./continuous_integration --use-ssl - - - name: Run durability test (plain) - run: | - cd tests/stress - source ve3/bin/activate - python3 durability --num-steps 5 - - - name: Run durability test (large) - run: | - cd tests/stress - source ve3/bin/activate - python3 durability --num-steps 20 - - release_jepsen_test: - name: "Release Jepsen Test" - runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl] - timeout-minutes: 60 - - steps: - - name: Set up repository - uses: actions/checkout@v4 - with: - # Number of commits to fetch. `0` indicates all history for all - # branches and tags. (default: 1) - fetch-depth: 0 - - - name: Build release binaries - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - # Initialize dependencies. - ./init - - # Build only memgraph release binary. - cd build - cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. - make -j$THREADS memgraph - - - name: Refresh Jepsen Cluster - run: | - cd tests/jepsen - ./run.sh cluster-refresh - - - name: Run Jepsen tests - run: | - cd tests/jepsen - ./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs - - - name: Save Jepsen report - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: "Jepsen Report-${{ env.OS }}" - path: tests/jepsen/Jepsen.tar.gz diff --git a/.github/workflows/reusable_release_tests.yaml b/.github/workflows/reusable_release_tests.yaml new file mode 100644 index 000000000..0bfe326e2 --- /dev/null +++ b/.github/workflows/reusable_release_tests.yaml @@ -0,0 +1,713 @@ +name: Reusable release tests + +on: + workflow_call: + inputs: + os: + type: string + description: "Target os. Default value is debian-11." + default: 'debian-12' + arch: + type: string + description: "Target architecture. Default value is amd." + default: 'amd' + runner_arch_label: + type: string + description: "Runner architecture label. Amd is built on X64, arm on ARM64. Default value is X64." + default: 'X64' + toolchain: + type: string + description: "Toolchain version. Default value is v5." + default: 'v5' + build_type: + type: string + description: "Memgraph Build type. Default value is Release." + default: 'Release' + threads: + type: number + description: "Number of threads to use for Memgraph build. Default value is 24." + default: 24 + +env: + OS: ${{ inputs.os }} + TOOLCHAIN: ${{ inputs.toolchain }} + ARCH: ${{ inputs.arch }} + THREADS: ${{ inputs.threads }} + MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }} + MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }} + BUILD_TYPE: ${{ inputs.build_type }} + +jobs: + community_build: + name: "Community build" + runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"] + timeout-minutes: 60 + + steps: + - name: Set up repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Spin up mgbuild container + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + run + + - name: Build community binary + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --build-type $BUILD_TYPE \ + --threads $THREADS \ + build-memgraph --community + + - name: Run unit tests + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --threads $THREADS \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph unit + + - name: Stop mgbuild container + if: always() + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + stop --remove + + coverage_build: + name: "Coverage build" + runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"] + timeout-minutes: 60 + env: + BUILD_TYPE: Debug + + steps: + - name: Set up repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Spin up mgbuild container + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + run + + - name: Build coverage binaries + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --build-type $BUILD_TYPE \ + --threads $THREADS \ + build-memgraph --coverage + + - name: Run unit tests + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph unit-coverage + + - name: Compute code coverage + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph code-coverage + + - name: Save code coverage + uses: actions/upload-artifact@v4 + with: + name: "Code coverage(Coverage build)-${{ env.OS }}" + path: tools/github/generated/code_coverage.tar.gz + + - name: Stop mgbuild container + if: always() + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + stop --remove + + debug_build: + name: "Debug build" + runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"] + timeout-minutes: 60 + env: + BUILD_TYPE: Debug + + steps: + - name: Set up repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Spin up mgbuild container + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + run + + - name: Build debug binary + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --build-type $BUILD_TYPE \ + --threads $THREADS \ + build-memgraph + + - name: Run leftover CTest tests + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --threads $THREADS \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph leftover-CTest + + - name: Run drivers tests + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --threads $THREADS \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph drivers + + - name: Run HA driver tests + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --threads $THREADS \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph drivers-high-availability + + - name: Run cppcheck and clang-format + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --threads $THREADS \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph cppcheck-and-clang-format + + - name: Save cppcheck and clang-format errors + uses: actions/upload-artifact@v4 + with: + name: "Code coverage(Debug build)-${{ env.OS }}" + path: tools/github/cppcheck_and_clang_format.txt + + - name: Stop mgbuild container + if: always() + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + stop --remove + + debug_integration_test: + name: "Debug integration tests" + runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"] + timeout-minutes: 60 + env: + BUILD_TYPE: Debug + + steps: + - name: Set up repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Spin up mgbuild container + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + run + + - name: Build debug binary + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --build-type $BUILD_TYPE \ + --threads $THREADS \ + build-memgraph + + - name: Run integration tests + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --threads $THREADS \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph integration + + - name: Stop mgbuild container + if: always() + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + stop --remove + + release_build: + name: "Release build" + runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"] + timeout-minutes: 60 + + steps: + - name: Set up repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Spin up mgbuild container + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + run + + - name: Build release binaries + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --build-type $BUILD_TYPE \ + --threads $THREADS \ + build-memgraph + + - name: Run GQL Behave tests + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph gql-behave + + - name: Save quality assurance status + uses: actions/upload-artifact@v4 + with: + name: "GQL Behave Status-${{ env.OS }}" + path: | + tests/gql_behave/gql_behave_status.csv + tests/gql_behave/gql_behave_status.html + + - name: Run unit tests + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --threads $THREADS \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph unit + + - name: Create enterprise DEB package + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + package-memgraph + + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + copy --package + + - name: Save enterprise DEB package + uses: actions/upload-artifact@v4 + with: + name: "Enterprise DEB package-${{ env.OS}}" + path: build/output/${{ env.OS }}/memgraph*.deb + + - name: Stop mgbuild container + if: always() + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + stop --remove + + release_benchmark_tests: + name: "Release Benchmark Tests" + runs-on: [self-hosted, Linux, X64, Debian10] + runs-on: [self-hosted, DockerMgBuild, Gen7, "${{ inputs.runner_arch_label }}"] + timeout-minutes: 60 + + steps: + - name: Set up repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Spin up mgbuild container + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + run + + - name: Build release binaries + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --build-type $BUILD_TYPE \ + --threads $THREADS \ + build-memgraph + + # ADAPT THIS + # - name: Run micro benchmark tests + # run: | + # # Activate toolchain. + # source /opt/toolchain-v4/activate + # + # # Run micro benchmark tests. + # cd build + # # The `eval` benchmark needs a large stack limit. + # ulimit -s 262144 + # ctest -R memgraph__benchmark -V + # + # - name: Run macro benchmark tests + # run: | + # cd tests/macro_benchmark + # ./harness QuerySuite MemgraphRunner \ + # --groups aggregation 1000_create unwind_create dense_expand match \ + # --no-strict + # + # - name: Run parallel macro benchmark tests + # run: | + # cd tests/macro_benchmark + # ./harness QueryParallelSuite MemgraphRunner \ + # --groups aggregation_parallel create_parallel bfs_parallel \ + # --num-database-workers 9 --num-clients-workers 30 \ + # --no-strict + + - name: Stop mgbuild container + if: always() + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + stop --remove + + release_e2e_test: + name: "Release End-to-end Test" + runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"] + timeout-minutes: 60 + + steps: + - name: Set up repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Spin up mgbuild container + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + run + + - name: Build release binaries + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --build-type $BUILD_TYPE \ + --threads $THREADS \ + build-memgraph + + # This step will be skipped because the e2e stream tests have been disabled + # We need to fix this as soon as possible + - name: Ensure Kafka and Pulsar are up + if: false + run: | + cd tests/e2e/streams/kafka + docker-compose up -d + cd ../pulsar + docker-compose up -d + + - name: Run e2e tests + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph e2e + + # Same as two steps prior + - name: Ensure Kafka and Pulsar are down + if: false + run: | + cd tests/e2e/streams/kafka + docker-compose down + cd ../pulsar + docker-compose down + + - name: Stop mgbuild container + if: always() + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + stop --remove + + release_durability_stress_tests: + name: "Release durability and stress tests" + runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"] + timeout-minutes: 60 + + steps: + - name: Set up repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Spin up mgbuild container + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + run + + - name: Build release binaries + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --build-type $BUILD_TYPE \ + --threads $THREADS \ + build-memgraph + + - name: Run stress test (plain) + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph stress-plain + + - name: Run stress test (SSL) + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph stress-ssl + + - name: Run durability test (plain) + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph durability + + - name: Run durability test (large) + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \ + --organization-name $MEMGRAPH_ORGANIZATION_NAME \ + test-memgraph durability-large + + - name: Stop mgbuild container + if: always() + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + stop --remove + + release_jepsen_test: + if: ${{ inputs.os == "debian-12" }} + name: "Release Jepsen Test" + runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"] + timeout-minutes: 60 + + steps: + - name: Set up repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Spin up mgbuild container + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + run + + - name: Build release binaries + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + --build-type $BUILD_TYPE \ + --threads $THREADS \ + build-memgraph + + - name: Copy memgraph binary + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + copy --binary + + - name: Refresh Jepsen Cluster + run: | + cd tests/jepsen + ./run.sh cluster-refresh + + - name: Run Jepsen tests + run: | + cd tests/jepsen + ./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs + + - name: Save Jepsen report + uses: actions/upload-artifact@v4 + if: always() + with: + name: "Jepsen Report-${{ env.OS }}" + path: tests/jepsen/Jepsen.tar.gz + + - name: Stop mgbuild container + if: always() + run: | + ./release/package/mgbuild.sh \ + --toolchain $TOOLCHAIN \ + --os $OS \ + --arch $ARCH \ + stop --remove diff --git a/release/package/mgbuild.sh b/release/package/mgbuild.sh index e24776f60..56ecca96c 100755 --- a/release/package/mgbuild.sh +++ b/release/package/mgbuild.sh @@ -436,6 +436,9 @@ test_memgraph() { durability) docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/stress && source ve3/bin/activate "'&& python3 durability --num-steps 5' ;; + durability-large) + docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/stress && source ve3/bin/activate "'&& python3 durability --num-steps 5' + ;; gql-behave) local test_output_dir="$MGBUILD_ROOT_DIR/tests/gql_behave" local test_output_host_dest="$PROJECT_ROOT/tests/gql_behave"