memgraph/.github/workflows/diff.yaml

name: Diff
concurrency:
  group: ${{ github.head_ref || github.sha }}
  cancel-in-progress: true

on:
  push:
    branches:
      - master
  workflow_dispatch:
  # pull_request:
  #   paths-ignore:
  #     - "docs/**"
  #     - "**/*.md"
  #     - ".clang-format"
  #     - "CODEOWNERS"
  #     - "licenses/*"

jobs:
  community_build:
    name: "Community build"
    runs-on: [self-hosted, Linux, X64, DockerMgBuild]
    timeout-minutes: 60
    env:
      THREADS: 24
      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
      OS: debian-11
      TOOLCHAIN: v5
      ARCH: amd
      BUILD_TYPE: RelWithDebInfo

    steps:
      - name: Set up repository
        uses: actions/checkout@v4
        with:
          # Number of commits to fetch. `0` indicates all history for all
          # branches and tags. (default: 1)
          fetch-depth: 0

      - name: Spin up mgbuild container
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          run

      - name: Build release binaries
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --build-type $BUILD_TYPE \
          --threads $THREADS \
          build-memgraph --community

      - name: Run unit tests
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --threads $THREADS \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph unit

      - name: Stop mgbuild container
        if: always()
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          stop --remove

  code_analysis:
    name: "Code analysis"
    runs-on: [self-hosted, Linux, X64, DockerMgBuild]
    timeout-minutes: 60
    env:
      THREADS: 24
      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
      OS: debian-11
      TOOLCHAIN: v5
      ARCH: amd
      BUILD_TYPE: Debug

    steps:
      - name: Set up repository
        uses: actions/checkout@v4
        with:
          # Number of commits to fetch. `0` indicates all history for all
          # branches and tags. (default: 1)
          fetch-depth: 0

      - name: Spin up mgbuild container
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          run

        # This is also needed if we want do to comparison against other branches
        # See https://github.community/t/checkout-code-fails-when-it-runs-lerna-run-test-since-master/17920
      - name: Fetch all history for all tags and branches
        run: git fetch

      - name: Initialize deps
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --build-type $BUILD_TYPE \
          --threads $THREADS \
          build-memgraph --init-only

      - name: Set base branch
        if: ${{ github.event_name == 'pull_request' }}
        run: |
          echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV

      - name: Set base branch # if we manually dispatch or push to master
        if: ${{ github.event_name != 'pull_request' }}
        run: |
          echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV

      - name: Python code analysis
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph code-analysis --base-branch "${{ env.BASE_BRANCH }}"

      - name: Build combined ASAN, UBSAN and coverage binaries
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --build-type $BUILD_TYPE \
          --threads $THREADS \
          build-memgraph --coverage --asan --ubsan

      - name: Run unit tests
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph unit-coverage

      - name: Compute code coverage
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph code-coverage

      - name: Save code coverage
        uses: actions/upload-artifact@v4
        with:
          name: "Code coverage(Code analysis)"
          path: tools/github/generated/code_coverage.tar.gz

      - name: Run clang-tidy
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --threads $THREADS \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph clang-tidy --base-branch "${{ env.BASE_BRANCH }}"

      - name: Stop mgbuild container
        if: always()
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          stop --remove

  debug_build:
    name: "Debug build"
    runs-on: [self-hosted, Linux, X64, DockerMgBuild]
    timeout-minutes: 100
    env:
      THREADS: 24
      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
      OS: debian-11
      TOOLCHAIN: v5
      ARCH: amd
      BUILD_TYPE: Debug

    steps:
      - name: Set up repository
        uses: actions/checkout@v4
        with:
          # Number of commits to fetch. `0` indicates all history for all
          # branches and tags. (default: 1)
          fetch-depth: 0

      - name: Spin up mgbuild container
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          run

      - name: Build release binaries
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --build-type $BUILD_TYPE \
          --threads $THREADS \
          build-memgraph

      - name: Run leftover CTest tests
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --threads $THREADS \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph leftover-CTest

      - name: Run drivers tests
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --threads $THREADS \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph drivers

      - name: Run HA driver tests
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --threads $THREADS \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph drivers-high-availability

      - name: Run integration tests
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --threads $THREADS \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph integration

      - name: Run cppcheck and clang-format
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --threads $THREADS \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph cppcheck-and-clang-format

      - name: Save cppcheck and clang-format errors
        uses: actions/upload-artifact@v4
        with:
          name: "Code coverage(Debug build)"
          path: tools/github/cppcheck_and_clang_format.txt

      - name: Stop mgbuild container
        if: always()
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          stop --remove

  release_build:
    name: "Release build"
    runs-on: [self-hosted, Linux, X64, DockerMgBuild]
    timeout-minutes: 100
    env:
      THREADS: 24
      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
      OS: debian-11
      TOOLCHAIN: v5
      ARCH: amd
      BUILD_TYPE: Release

    steps:
      - name: Set up repository
        uses: actions/checkout@v4
        with:
          # Number of commits to fetch. `0` indicates all history for all
          # branches and tags. (default: 1)
          fetch-depth: 0

      - name: Spin up mgbuild container
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          run

      - name: Build release binaries
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --build-type $BUILD_TYPE \
          --threads $THREADS \
          build-memgraph

      - name: Run GQL Behave tests
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph gql-behave

      - name: Save quality assurance status
        uses: actions/upload-artifact@v4
        with:
          name: "GQL Behave Status"
          path: |
            tests/gql_behave/gql_behave_status.csv
            tests/gql_behave/gql_behave_status.html

      - name: Run unit tests
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --threads $THREADS \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph unit

      # This step will be skipped because the e2e stream tests have been disabled
      # We need to fix this as soon as possible
      - name: Ensure Kafka and Pulsar are up
        if: false
        run: |
          cd tests/e2e/streams/kafka
          docker-compose up -d
          cd ../pulsar
          docker-compose up -d

      - name: Run e2e tests
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph e2e

      # Same as two steps prior
      - name: Ensure Kafka and Pulsar are down
        if: false
        run: |
          cd tests/e2e/streams/kafka
          docker-compose down
          cd ../pulsar
          docker-compose down

      - name: Run stress test (plain)
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph stress-plain

      - name: Run stress test (SSL)
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph stress-ssl

      - name: Run durability test
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph durability

      - name: Create enterprise DEB package
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          package-memgraph

          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          copy --package

      - name: Save enterprise DEB package
        uses: actions/upload-artifact@v4
        with:
          name: "Enterprise DEB package"
          path: build/output/${{ env.OS }}/memgraph*.deb

      - name: Copy build logs
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          copy --build-logs

      - name: Save test data
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: "Test data(Release build)"
          path: build/logs

      - name: Stop mgbuild container
        if: always()
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          stop --remove

  release_jepsen_test:
    name: "Release Jepsen Test"
    runs-on: [self-hosted, Linux, X64, DockerMgBuild]
    timeout-minutes: 80
    env:
      THREADS: 24
      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
      OS: debian-12
      TOOLCHAIN: v5
      ARCH: amd
      BUILD_TYPE: RelWithDebInfo

    steps:
      - name: Set up repository
        uses: actions/checkout@v4
        with:
          # Number of commits to fetch. `0` indicates all history for all
          # branches and tags. (default: 1)
          fetch-depth: 0

      - name: Spin up mgbuild container
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          run

      - name: Build release binaries
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --build-type $BUILD_TYPE \
          --threads $THREADS \
          build-memgraph

      - name: Copy memgraph binary
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          copy --binary

      - name: Refresh Jepsen Cluster
        run: |
          cd tests/jepsen
          ./run.sh cluster-refresh

      - name: Run Jepsen tests
        run: |
          cd tests/jepsen
          ./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs

      - name: Save Jepsen report
        uses: actions/upload-artifact@v4
        if: ${{ always() }}
        with:
          name: "Jepsen Report"
          path: tests/jepsen/Jepsen.tar.gz

      - name: Stop mgbuild container
        if: always()
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          stop --remove

  release_benchmarks:
    name: "Release benchmarks"
    runs-on: [self-hosted, Linux, X64, DockerMgBuild, Gen7]
    timeout-minutes: 60
    env:
      THREADS: 24
      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
      OS: debian-11
      TOOLCHAIN: v5
      ARCH: amd
      BUILD_TYPE: Release

    steps:
      - name: Set up repository
        uses: actions/checkout@v4
        with:
          # Number of commits to fetch. `0` indicates all history for all
          # branches and tags. (default: 1)
          fetch-depth: 0

      - name: Spin up mgbuild container
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          run

      - name: Build release binaries
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --build-type $BUILD_TYPE \
          --threads $THREADS \
          build-memgraph

      - name: Run macro benchmarks
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph macro-benchmark

      - name: Get branch name (merge)
        if: github.event_name != 'pull_request'
        shell: bash
        run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV

      - name: Get branch name (pull request)
        if: github.event_name == 'pull_request'
        shell: bash
        run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV

      - name: Upload macro benchmark results
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph upload-to-bench-graph \
          --benchmark-name "macro_benchmark" \
          --benchmark-results "../../tests/macro_benchmark/.harness_summary" \
          --github-run-id ${{ github.run_id }} \
          --github-run-number ${{ github.run_number }} \
          --head-branch-name ${{ env.BRANCH_NAME }}

      - name: Run mgbench
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph mgbench

      - name: Upload mgbench results
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          --enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
          --organization-name $MEMGRAPH_ORGANIZATION_NAME \
          test-memgraph upload-to-bench-graph \
          --benchmark-name "mgbench" \
          --benchmark-results "../../tests/mgbench/benchmark_result.json" \
          --github-run-id "${{ github.run_id }}" \
          --github-run-number "${{ github.run_number }}" \
          --head-branch-name "${{ env.BRANCH_NAME }}"

      - name: Stop mgbuild container
        if: always()
        run: |
          ./release/package/mgbuild.sh \
          --toolchain $TOOLCHAIN \
          --os $OS \
          --arch $ARCH \
          stop --remove