tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into T0515-MG-fixed-size-pool-resource

Browse Source
This commit is contained in:
Antonio Andelic 2021-06-17 11:31:41 +02:00
commit 51b5d81018
212 changed files with 11853 additions and 4109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.clang-tidy
View File

@ -26,6 +26,7 @@ Checks: '*,
-fuchsia-virtual-inheritance,
-google-explicit-constructor,
-google-readability-*,
-google-runtime-references,
-hicpp-avoid-c-arrays,
-hicpp-avoid-goto,
-hicpp-braces-around-statements,
@ -52,9 +53,10 @@ Checks: '*,
-readability-else-after-return,
-readability-implicit-bool-conversion,
-readability-magic-numbers,
-readability-named-parameter'
-readability-named-parameter,
-misc-no-recursion'
WarningsAsErrors: ''
HeaderFilterRegex: ''
HeaderFilterRegex: 'src/.*'
AnalyzeTemporaryDtors: false
FormatStyle: none
CheckOptions:

68
.github/workflows/daily_banchmark.yaml vendored Normal file
View File

@ -0,0 +1,68 @@
name: Daily Benchmark
on:
workflow_dispatch:
schedule:
- cron: "0 1 * * *"
jobs:
release_benchmarks:
name: "Release benchmarks"
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
env:
THREADS: 24
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build only memgraph release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Run macro benchmarks
run: |
cd tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Upload macro benchmark results
run: |
cd tools/bench-graph-client
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "macro_benchmark" \
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"
- name: Run mgbench
run: |
cd tests/mgbench
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
- name: Upload mgbench results
run: |
cd tools/bench-graph-client
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "mgbench" \
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"

180
.github/workflows/diff.yaml vendored
View File

@ -5,12 +5,13 @@ on:
paths-ignore:
- 'docs/**'
- '**/*.md'
- '.clang-*'
- '.clang-format'
- 'CODEOWNERS'
jobs:
community_build:
name: "Community build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
@ -42,7 +43,7 @@ jobs:
# Run unit tests.
cd build
ctest -R memgraph__unit --output-on-failure
ctest -R memgraph__unit --output-on-failure -j$THREADS
- name: Run stress test
run: |
@ -54,8 +55,13 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v2/activate
# Create community DEB package.
cd build
# create mgconsole
# we use the -B to force the build
make -j$THREADS -B mgconsole
# Create community DEB package.
mkdir output && cd output
cpack -G DEB --config ../CPackConfig.cmake
@ -65,9 +71,9 @@ jobs:
name: "Community DEB package"
path: build/output/memgraph*.deb
coverage_build:
name: "Coverage build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
code_analysis:
name: "Code analysis"
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
@ -79,7 +85,7 @@ jobs:
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build coverage binaries
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
@ -87,9 +93,8 @@ jobs:
# Initialize dependencies.
./init
# Build coverage binaries.
cd build
cmake -DTEST_COVERAGE=ON ..
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
make -j$THREADS memgraph__unit
- name: Run unit tests
@ -97,9 +102,9 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v2/activate
# Run unit tests.
# Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
cd build
ctest -R memgraph__unit --output-on-failure
LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
- name: Compute code coverage
run: |
@ -120,9 +125,19 @@ jobs:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
- name: Run clang-tidy
run: |
source /opt/toolchain-v2/activate
# Restrict clang-tidy results only to the modified parts
git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
# Fail if any warning is reported
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
debug_build:
name: "Debug build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
@ -196,7 +211,7 @@ jobs:
release_build:
name: "Release build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
@ -208,21 +223,6 @@ jobs:
# branches and tags. (default: 1)
fetch-depth: 0
- name: Set up parent
run: |
# Remove parent folder (if it exists).
cd ..
if [ -d parent ]; then
rm -rf parent
fi
# Copy untouched repository to parent folder.
cp -r memgraph parent
# Checkout previous commit
cd parent
git checkout HEAD~1
- name: Build release binaries
run: |
# Activate toolchain.
@ -236,47 +236,6 @@ jobs:
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Build parent binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
cd ../parent
./init
# Build parent binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS memgraph memgraph__macro_benchmark
- name: Run macro benchmark tests
run: |
cd tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Run parent macro benchmark tests
run: |
cd ../parent/tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Compute macro benchmark summary
run: |
./tools/github/macro_benchmark_summary \
--current tests/macro_benchmark/.harness_summary \
--previous ../parent/tests/macro_benchmark/.harness_summary \
--output macro_benchmark_summary.txt
- name: Save macro benchmark summary
uses: actions/upload-artifact@v2
with:
name: "Macro benchmark summary"
path: macro_benchmark_summary.txt
- name: Run GQL Behave tests
run: |
cd tests/gql_behave
@ -290,14 +249,14 @@ jobs:
tests/gql_behave/gql_behave_status.csv
tests/gql_behave/gql_behave_status.html
- name: Run e2e replication tests
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
- name: Run stress test (plain)
run: |
@ -320,8 +279,13 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v2/activate
# Create enterprise DEB package.
cd build
# create mgconsole
# we use the -B to force the build
make -j$THREADS -B mgconsole
# Create enterprise DEB package.
mkdir output && cd output
cpack -G DEB --config ../CPackConfig.cmake
@ -331,6 +295,15 @@ jobs:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Save test data
uses: actions/upload-artifact@v2
if: always()
with:
name: "Test data"
path: |
# multiple paths could be defined
build/logs
release_jepsen_test:
name: "Release Jepsen Test"
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
@ -370,3 +343,64 @@ jobs:
with:
name: "Jepsen Report"
path: tests/jepsen/Jepsen.tar.gz
release_benchmarks:
name: "Release benchmarks"
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
env:
THREADS: 24
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build only memgraph release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Run macro benchmarks
run: |
cd tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Upload macro benchmark results
run: |
cd tools/bench-graph-client
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "macro_benchmark" \
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"
- name: Run mgbench
run: |
cd tests/mgbench
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
- name: Upload mgbench results
run: |
cd tools/bench-graph-client
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "mgbench" \
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"

44
.github/workflows/full_clang_tidy.yaml vendored Normal file
View File

@ -0,0 +1,44 @@
name: Run clang-tidy on the full codebase
on:
workflow_dispatch:
jobs:
clang_tidy_check:
name: "Clang-tidy check"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
env:
THREADS: 24
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build debug binaries.
cd build
cmake ..
make -j$THREADS
- name: Run clang-tidy
run: |
source /opt/toolchain-v2/activate
# The results are also written to standard output in order to retain them in the logs
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v2/bin/clang-tidy "$PWD/src/*" |
tee ./build/full_clang_tidy_output.txt
- name: Summarize clang-tidy results
run: cat ./build/full_clang_tidy_output.txt | ./tools/github/clang-tidy/count_errors.sh

248
.github/workflows/package_all.yaml vendored Normal file
View File

@ -0,0 +1,248 @@
name: Package All
# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
on: workflow_dispatch
jobs:
centos-7_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community centos-7
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-7_community
path: build/output/centos-7/memgraph*.rpm
centos-8_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community centos-8
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-8_community
path: build/output/centos-8/memgraph*.rpm
debian-9_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community debian-9
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-9_community
path: build/output/debian-9/memgraph*.deb
debian-10_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community debian-10
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-10_community
path: build/output/debian-10/memgraph*.deb
docker_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
cd release/package
./run.sh package community debian-10 --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: docker_community
path: build/output/docker/memgraph*.tar.gz
ubuntu-1804_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community ubuntu-18.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-1804_community
path: build/output/ubuntu-18.04/memgraph*.deb
ubuntu-2004_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community ubuntu-20.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-2004_community
path: build/output/ubuntu-20.04/memgraph*.deb
centos-7_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise centos-7
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-7_enterprise
path: build/output/centos-7/memgraph*.rpm
centos-8_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise centos-8
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-8_enterprise
path: build/output/centos-8/memgraph*.rpm
debian-9_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise debian-9
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-9_enterprise
path: build/output/debian-9/memgraph*.deb
debian-10_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise debian-10
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-10_enterprise
path: build/output/debian-10/memgraph*.deb
docker_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
cd release/package
./run.sh package enterprise debian-10 --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: docker_enterprise
path: build/output/docker/memgraph*.tar.gz
ubuntu-1804_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise ubuntu-18.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-1804_enterprise
path: build/output/ubuntu-18.04/memgraph*.deb
ubuntu-2004_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise ubuntu-20.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-2004_enterprise
path: build/output/ubuntu-20.04/memgraph*.deb

18
.github/workflows/release_centos8.yaml vendored
View File

@ -39,8 +39,13 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v2/activate
# Create community RPM package.
cd build
# create mgconsole
# we use the -B to force the build
make -j$THREADS -B mgconsole
# Create community RPM package.
mkdir output && cd output
cpack -G RPM --config ../CPackConfig.cmake
rpmlint memgraph*.rpm
@ -232,8 +237,13 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v2/activate
# Create enterprise RPM package.
cd build
# create mgconsole
# we use the -B to force the build
make -j$THREADS -B mgconsole
# Create enterprise RPM package.
mkdir output && cd output
cpack -G RPM --config ../CPackConfig.cmake
rpmlint memgraph*.rpm
@ -283,14 +293,14 @@ jobs:
tests/gql_behave/gql_behave_status.csv
tests/gql_behave/gql_behave_status.html
- name: Run e2e replication tests
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
- name: Run stress test (plain)
run: |

20
.github/workflows/release_debian10.yaml vendored
View File

@ -1,4 +1,4 @@
name: Release Debian10
name: Release Debian 10
on:
workflow_dispatch:
@ -39,8 +39,13 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v2/activate
# Create community DEB package.
cd build
# create mgconsole
# we use the -B to force the build
make -j$THREADS -B mgconsole
# Create community DEB package.
mkdir output && cd output
cpack -G DEB --config ../CPackConfig.cmake
@ -231,8 +236,13 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v2/activate
# Create enterprise DEB package.
cd build
# create mgconsole
# we use the -B to force the build
make -j$THREADS -B mgconsole
# Create enterprise DEB package.
mkdir output && cd output
cpack -G DEB --config ../CPackConfig.cmake
@ -281,14 +291,14 @@ jobs:
tests/gql_behave/gql_behave_status.csv
tests/gql_behave/gql_behave_status.html
- name: Run e2e replication tests
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
- name: Run stress test (plain)
run: |

20
.github/workflows/release_ubuntu2004.yaml vendored
View File

@ -1,4 +1,4 @@
name: Release Ubuntu20.04
name: Release Ubuntu 20.04
on:
workflow_dispatch:
@ -39,8 +39,13 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v2/activate
# Create community DEB package.
cd build
# create mgconsole
# we use the -B to force the build
make -j$THREADS -B mgconsole
# Create community DEB package.
mkdir output && cd output
cpack -G DEB --config ../CPackConfig.cmake
@ -231,8 +236,13 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v2/activate
# Create enterprise DEB package.
cd build
# create mgconsole
# we use the -B to force the build
make -j$THREADS -B mgconsole
# Create enterprise DEB package.
mkdir output && cd output
cpack -G DEB --config ../CPackConfig.cmake
@ -281,14 +291,14 @@ jobs:
tests/gql_behave/gql_behave_status.csv
tests/gql_behave/gql_behave_status.html
- name: Run e2e replication tests
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
- name: Run stress test (plain)
run: |

464
CHANGELOG.md
View File

@ -1,461 +1,5 @@
# Change Log
Change Log for all versions of Memgraph can be found on-line at
https://docs.memgraph.com/memgraph/changelog
## Future
### Major Feature and Improvements
* Added replication to community version.
* Add support for multiple query modules directories at the same time.
You can now define multiple, comma-separated paths to directories from
which the modules will be loaded using the `--query-modules-directory` flag.
### Bug Fixes
* Fixed garbage collector by correctly marking the oldest current timestamp
after the database was recovered using the durability files.
## v1.3.0
### Breaking Changes
* Added extra information in durability files to support replication, making it
incompatible with the durability files generated by older versions of
Memgraph. Even though the replication is an Enterprise feature, the files are
compatible with the Community version.
### Major Features and Improvements
* Added support for data replication across a cluster of Memgraph instances.
Supported instance types are MAIN and REPLICA. Supported replication modes
are SYNC (all SYNC REPLICAS have to receive data before the MAIN can commit
the transaction), ASYNC (MAIN doesn't care if data is replicated), SYNC WITH
TIMEOUT (MAIN will wait for REPLICAS within the given timeout period, after
timout, replication isn't aborted but the replication demotes the REPLICA to
the ASYNC mode).
* Added support for query type deduction. Possible query types are `r` (read),
`w` (write), `rw` (read-write). The query type is returned as a part of the
summary.
* Improved logging capabilities by introducing granular logging levels. Added
new flag, `--log-level`, which specifies the minimum log level that will be
printed. E.g., it's possible to print incoming queries or Bolt server states.
* Added ability to lock the storage data directory by executing the `LOCK DATA
DIRECTORY;` query which delays the deletion of the files contained in the
data directory. The data directory can be unlocked again by executing the
`UNLOCK DATA DIRECTORY;` query.
### Bug Fixes and Other Changes
* Added cleanup of query executions if not in an explicit transaction.
* Fix RPC dangling reference.
## v1.2.0
### Breaking Changes
* SSL is disabled by default (`--bolt-cert-file` and `--bolt-key-file` are
empty). This change might only affect the client connection configuration.
### Major Features and Improvements
* Added support for Bolt v4.0 and v4.1.
* Added `mgp_networkx.py` as an alternative implementation of NetworkX graph
objects, which is useful to use Memgraph data from NetworkX algorithms
optimally.
* Added `nxalg.py` query module as a proxy to NetworkX algorithms.
* Added plan optimization to use a label-property index where the property is
not null. As a result, the query engine, instead of scanning all elements and
applying the filter, performs a label-property index lookup when possible.
### Bug Fixes and Other Changes
* Fixed Cypher `ID` function `Null` handling. When the `ID` function receives
`Null`, it will also return `Null`.
* Fixed bug that caused random crashes in SSL communication on platforms
that use older versions of OpenSSL (< 1.1) by adding proper multi-threading
handling.
* Fix `DISCARD` message handling. The query is now executed before discarding
the results.
## v1.1.0
### Major Features and Improvements
* Properties in nodes and edges are now stored encoded and compressed. This
change significantly reduces memory usage. Depending on the specific dataset,
total memory usage can be reduced up to 50%.
* Added support for rescanning query modules. Previously, the query modules
directory was scanned only upon startup. Now it is scanned each time the user
requests to load a query module. The functions used to load the query modules
were renamed to `mg.load()` and `mg.load_all()` (from `mg.reload()` and
`mg.reload_all()`).
* Improved execution performance of queries that have an IN list filter by
using label+property indices.
Example: `MATCH (n:Label) WHERE n.property IN [] ...`
* Added support for `ANY` and `NONE` openCypher functions. Previously, only
`ALL` and `SINGLE` functions were implemented.
### Bug Fixes and Other Changes
* Fixed invalid paths returned by variable expansion when the starting node and
destination node used the same symbol.
Example: `MATCH path = (n:Person {name: "John"})-[:KNOWS*]->(n) RETURN path`
* Improved semantics of `ALL` and `SINGLE` functions to be consistent with
openCypher when handling lists with `Null`s.
* `SHOW CONSTRAINT INFO` now returns property names as a list for unique
constraints.
* Escaped label/property/edgetype names in `DUMP DATABASE` to support names
with spaces in them.
* Fixed handling of `DUMP DATABASE` queries in multi-command transactions
(`BEGIN`, ..., `COMMIT`).
* Fixed handling of various query types in explicit transactions. For example,
constraints were allowed to be created in multi-command transactions
(`BEGIN`, ..., `COMMIT`) but that isn't a transactional operation and as such
can't be allowed in multi-command transactions.
* Fixed integer overflow bugs in `COUNT`, `LIMIT` and `SKIP`.
* Fixed integer overflow bugs in weighted shortest path expansions.
* Fixed various other integer overflow bugs in query execution.
* Added Marvel Comic Universe tutorial.
* Added FootballTransfers tutorial.
## v1.0.0
### Major Features and Improvements
* [Enterprise Ed.] Exposed authentication username/rolename regex as a flag
(`--auth-user-or-role-name-regex`).
* [Enterprise Ed.] Improved auth module error handling and added support for
relative paths.
* Added support for Python query modules. This release of Memgraph supports
query modules written using the already existing C API and the new Python
API.
* Added support for unique constraints. The unique constraint is created with a
label and one or more properties.
* Implemented support for importing CSV files (`mg_import_csv`). The importer
is compatible with the Neo4j batch CSV importer.
* Snapshot and write-ahead log format changed (backward compatible with v0.50).
* Vertices looked up by their openCypher ID (`MATCH (n) WHERE ID(n) = ...`)
will now find the node in O(logn) instead of O(n).
* Improved planning of BFS expansion, a faster, specific approach is now
favored instead of a ScanAll+Filter operation.
* Added syntax for limiting memory of `CALL`.
* Exposed server name that should be used for Bolt handshake as flag
(`--bolt-server-name-for-init`).
* Added several more functions to the query module C API.
* Implemented a storage locking mechanism that prevents the user from
concurrently starting two Memgraph instances with the same data directory.
### Bug Fixes and Other Changes
* [Enterprise Ed.] Fixed a bug that crashed the database when granting
privileges to a user.
* [Enterprise Ed.] Improved Louvain algorithm for community detection.
* Type of variable expansion is now printed in `EXPLAIN` (e.g. ExpandVariable,
STShortestPath, BFSExpand, WeightedShortestPath).
* Correctly display `CALL` in `EXPLAIN` output.
* Correctly delimit arguments when printing the signature of a query module.
* Fixed a planning issue when `CALL` preceded filtering.
* Fixed spelling mistakes in the storage durability module.
* Fixed storage GC indices/constraints subtle race condition.
* Reduced memory allocations in storage API and indices.
* Memgraph version is now outputted to `stdout` when Memgraph is started.
* Improved RPM packaging.
* Reduced number of errors reported in production log when loading query
modules.
* Removed `early access` wording from the Community Offering license.
## v0.50.0
### Breaking Changes
* [Enterprise Ed.] Remove support for Kafka streams.
* Snapshot and write-ahead log format changed (not backward compatible).
* Removed support for unique constraints.
* Label indices aren't created automatically, create them explicitly instead.
* Renamed several database flags. Please see the configuration file for a list of current flags.
### Major Features and Improvements
* [Enterprise Ed.] Add support for auth module.
* [Enterprise Ed.] LDAP support migrated to auth module.
* Implemented new graph storage engine.
* Add support for disabling properties on edges.
* Add support for existence constraints.
* Add support for custom openCypher procedures using a C API.
* Support loading query modules implementing read-only procedures.
* Add `CALL <procedure> YIELD <result>` syntax for invoking loaded procedures.
* Add `CREATE INDEX ON :Label` for creating label indices.
* Add `DROP INDEX ON :Label` for dropping label indices.
* Add `DUMP DATABASE` clause to openCypher.
* Add functions for treating character strings as byte strings.
### Bug Fixes and Other Changes
* Fix several memory management bugs.
* Reduce memory usage in query execution.
* Fix bug that crashes the database when `EXPLAIN` is used.
## v0.15.0
### Breaking Changes
* Snapshot and write-ahead log format changed (not backward compatible).
* `indexInfo()` function replaced with `SHOW INDEX INFO` syntax.
* Removed support for unique index. Use unique constraints instead.
* `CREATE UNIQUE INDEX ON :label (property)` replaced with `CREATE CONSTRAINT ON (n:label) ASSERT n.property IS UNIQUE`.
* Changed semantics for `COUNTER` openCypher function.
### Major Features and Improvements
* [Enterprise Ed.] Add new privilege, `STATS` for accessing storage info.
* [Enterprise Ed.] LDAP authentication and authorization support.
* [Enterprise Ed.] Add audit logging feature.
* Add multiple properties unique constraint which replace unique indices.
* Add `SHOW STORAGE INFO` feature.
* Add `PROFILE` clause to openCypher.
* Add `CREATE CONSTRAINT` clause to openCypher.
* Add `DROP CONSTRAINT` clause to openCypher.
* Add `SHOW CONSTRAINT INFO` feature.
* Add `uniformSample` function to openCypher.
* Add regex matching to openCypher.
### Bug Fixes and Other Changes
* Fix bug in explicit transaction handling.
* Fix bug in edge filtering by edge type and destination.
* Fix bug in query comment parsing.
* Fix bug in query symbol table.
* Fix OpenSSL memory leaks.
* Make authentication case insensitive.
* Remove `COALESCE` function.
* Add movie tutorial.
* Add backpacking tutorial.
## v0.14.0
### Breaking Changes
* Write-ahead log format changed (not backward compatible).
### Major Features and Improvements
* [Enterprise Ed.] Reduce memory usage in distributed usage.
* Add `DROP INDEX` feature.
* Improve SSL error messages.
### Bug Fixes and Other Changes
* [Enterprise Ed.] Fix issues with reading and writing in a distributed query.
* Correctly handle an edge case with unique constraint checks.
* Fix a minor issue with `mg_import_csv`.
* Fix an issue with `EXPLAIN`.
## v0.13.0
### Breaking Changes
* Write-ahead log format changed (not backward compatible).
* Snapshot format changed (not backward compatible).
### Major Features and Improvements
* [Enterprise Ed.] Authentication and authorization support.
* [Enterprise Ed.] Kafka integration.
* [Enterprise Ed.] Support dynamic worker addition in distributed.
* Reduce memory usage and improve overall performance.
* Add `CREATE UNIQUE INDEX` clause to openCypher.
* Add `EXPLAIN` clause to openCypher.
* Add `inDegree` and `outDegree` functions to openCypher.
* Improve BFS performance when both endpoints are known.
* Add new `node-label`, `relationship-type` and `quote` options to
`mg_import_csv` tool.
* Reduce memory usage of `mg_import_csv`.
### Bug Fixes and Other Changes
* [Enterprise Ed.] Fix an edge case in distributed index creation.
* [Enterprise Ed.] Fix issues with Cartesian in distributed queries.
* Correctly handle large messages in Bolt protocol.
* Fix issues when handling explicitly started transactions in queries.
* Allow openCypher keywords to be used as variable names.
* Revise and make user visible error messages consistent.
* Improve aborting time consuming execution.
## v0.12.0
### Breaking Changes
* Snapshot format changed (not backward compatible).
### Major Features and Improvements
* Improved Id Cypher function.
* Added string functions to openCypher (`lTrim`, `left`, `rTrim`, `replace`,
`reverse`, `right`, `split`, `substring`, `toLower`, `toUpper`, `trim`).
* Added `timestamp` function to openCypher.
* Added support for dynamic property access with `[]` operator.
## v0.11.0
### Major Features and Improvements
* [Enterprise Ed.] Improve Cartesian support in distributed queries.
* [Enterprise Ed.] Improve distributed execution of BFS.
* [Enterprise Ed.] Dynamic graph partitioner added.
* Static nodes/edges id generators exposed through the Id Cypher function.
* Properties on disk added.
* Telemetry added.
* SSL support added.
* `toString` function added.
### Bug Fixes and Other Changes
* Document issues with Docker on OS X.
* Add BFS and Dijkstra's algorithm examples to documentation.
## v0.10.0
### Breaking Changes
* Snapshot format changed (not backward compatible).
### Major Features and Improvements
* [Enterprise Ed.] Distributed storage and execution.
* `reduce` and `single` functions added to openCypher.
* `wShortest` edge expansion added to openCypher.
* Support packaging RPM on CentOS 7.
### Bug Fixes and Other Changes
* Report an error if updating a deleted element.
* Log an error if reading info on available memory fails.
* Fix a bug when `MATCH` would stop matching if a result was empty, but later
results still contain data to be matched. The simplest case of this was the
query: `UNWIND [1,2,3] AS x MATCH (n :Label {prop: x}) RETURN n`. If there
was no node `(:Label {prop: 1})`, then the `MATCH` wouldn't even try to find
for `x` being 2 or 3.
* Report an error if trying to compare a property value with something that
cannot be stored in a property.
* Fix crashes in some obscure cases.
* Commit log automatically garbage collected.
* Add minor performance improvements.
## v0.9.0
### Breaking Changes
* Snapshot format changed (not backward compatible).
* Snapshot configuration flags changed, general durability flags added.
### Major Features and Improvements
* Write-ahead log added.
* `nodes` and `relationships` functions added.
* `UNION` and `UNION ALL` is implemented.
* Concurrent index creation is now enabled.
### Bug Fixes and Other Changes
## v0.8.0
### Major Features and Improvements
* CASE construct (without aggregations).
* Named path support added.
* Maps can now be stored as node/edge properties.
* Map indexing supported.
* `rand` function added.
* `assert` function added.
* `counter` and `counterSet` functions added.
* `indexInfo` function added.
* `collect` aggregation now supports Map collection.
* Changed the BFS syntax.
### Bug Fixes and Other Changes
* Use \u to specify 4 digit codepoint and \U for 8 digit
* Keywords appearing in header (named expressions) keep original case.
* Our Bolt protocol implementation is now completely compatible with the protocol version 1 specification. (https://boltprotocol.org/v1/)
* Added a log warning when running out of memory and the `memory_warning_threshold` flag
* Edges are no longer additionally filtered after expansion.
## v0.7.0
### Major Features and Improvements
* Variable length path `MATCH`.
* Explicitly started transactions (multi-query transactions).
* Map literal.
* Query parameters (except for parameters in place of property maps).
* `all` function in openCypher.
* `degree` function in openCypher.
* User specified transaction execution timeout.
### Bug Fixes and Other Changes
* Concurrent `BUILD INDEX` deadlock now returns an error to the client.
* A `MATCH` preceeded by `OPTIONAL MATCH` expansion inconsistencies.
* High concurrency Antlr parsing bug.
* Indexing improvements.
* Query stripping and caching speedups.
## v0.6.0
### Major Features and Improvements
* AST caching.
* Label + property index support.
* Different logging setup & format.
## v0.5.0
### Major Features and Improvements
* Use label indexes to speed up querying.
* Generate multiple query plans and use the cost estimator to select the best.
* Snapshots & Recovery.
* Abandon old yaml configuration and migrate to gflags.
* Query stripping & AST caching support.
### Bug Fixes and Other Changes
* Fixed race condition in MVCC. Hints exp+aborted race condition prevented.
* Fixed conceptual bug in MVCC GC. Evaluate old records w.r.t. the oldest.
transaction's id AND snapshot.
* User friendly error messages thrown from the query engine.
## Build 837
### Bug Fixes and Other Changes
* List indexing supported with preceeding IN (for example in query `RETURN 1 IN [[1,2]][0]`).
## Build 825
### Major Features and Improvements
* RETURN *, count(*), OPTIONAL MATCH, UNWIND, DISTINCT (except DISTINCT in aggregate functions), list indexing and slicing, escaped labels, IN LIST operator, range function.
### Bug Fixes and Other Changes
* TCP_NODELAY -> import should be faster.
* Clear hint bits.
## Build 783
### Major Features and Improvements
* SKIP, LIMIT, ORDER BY.
* Math functions.
* Initial support for MERGE clause.
### Bug Fixes and Other Changes
* Unhandled Lock Timeout Exception.
## Build 755
### Major Features and Improvements
* MATCH, CREATE, WHERE, SET, REMOVE, DELETE.
All the updates to the Change Log can be made in the following repository:
https://github.com/memgraph/docs

9
CMakeLists.txt
View File

@ -312,8 +312,9 @@ if (UBSAN)
# runtime library and c++ standard libraries are present.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1
# Make sure llvm-symbolizer binary is in path
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
# Make sure llvm-symbolizer binary is in path.
# To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
endif()
set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact python version used by the query modules")
@ -335,3 +336,7 @@ endif()
if(QUERY_MODULES)
add_subdirectory(query_modules)
endif()
install(FILES ${CMAKE_BINARY_DIR}/bin/mgconsole
PERMISSIONS OWNER_EXECUTE OWNER_READ OWNER_WRITE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
TYPE BIN)

5
CODEOWNERS
View File

@ -1,4 +1 @@
/docs/ @gitbuda
/src/communication/ @antonio2368
/src/query/ @the-joksim
/src/storage/ @antonio2368
* @gitbuda @antonio2368 @antaljanosbenjamin @kostasrim

8
config/flags.yaml
View File

@ -83,6 +83,14 @@ modifications:
value: "/usr/lib/memgraph/auth_module/example.py"
override: false
- name: "memory_limit"
value: "0"
override: true
- name: "isolation_level"
value: "SNAPSHOT_ISOLATION"
override: true
undocumented:
- "flag_file"
- "also_log_to_stderr"

17
environment/os/centos-7.sh
View File

@ -18,6 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-devel libxml2-devel perl-Digest-MD5 # llvm
libedit-devel pcre-devel automake bison # swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
readline # for cmake and llvm
libffi libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconfig # build system
@ -46,10 +48,13 @@ MEMGRAPH_BUILD_DEPS=(
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
@ -74,16 +79,13 @@ check() {
exit 1
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
if [ "$SUDO_USER" == "" ]; then
echo "Please run as sudo."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
@ -117,11 +119,16 @@ install() {
continue
fi
if [ "$pkg" == PyYAML ]; then
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
continue
fi
yum install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

18
environment/os/centos-8.sh
View File

@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
@ -25,6 +26,7 @@ TOOLCHAIN_RUN_DEPS=(
readline # for cmake and llvm
libffi libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
@ -45,10 +47,13 @@ MEMGRAPH_BUILD_DEPS=(
doxygen graphviz # source documentation generators
which mono-complete dotnet-sdk-3.1 nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
@ -67,16 +72,13 @@ check() {
exit 1
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
if [ "$SUDO_USER" == "" ]; then
echo "Please run as sudo."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
@ -85,6 +87,7 @@ install() {
echo "NOTE: export LANG=en_US.utf8"
fi
dnf install -y epel-release
dnf install -y 'dnf-command(config-manager)'
dnf config-manager --set-enabled powertools # Required to install texinfo.
dnf update -y
dnf install -y wget git python36 python3-pip
@ -134,11 +137,16 @@ install() {
continue
fi
if [ "$pkg" == PyYAML ]; then
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
continue
fi
dnf install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

7
environment/os/debian-10.sh
View File

@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
@ -43,13 +45,17 @@ MEMGRAPH_BUILD_DEPS=(
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
cat >/etc/apt/sources.list <<EOF
deb http://deb.debian.org/debian/ buster main non-free contrib
@ -82,5 +88,6 @@ EOF
apt install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

7
environment/os/debian-9.sh
View File

@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
@ -41,15 +43,20 @@ MEMGRAPH_BUILD_DEPS=(
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
install_all_apt "$1"
}
deps=$2"[*]"
"$1" "${!deps}"

6
environment/os/template.sh
View File

@ -8,23 +8,29 @@ source "$DIR/../util.sh"
TOOLCHAIN_BUILD_DEPS=(
pkg
)
TOOLCHAIN_RUN_DEPS=(
pkg
)
MEMGRAPH_BUILD_DEPS=(
pkg
)
list() {
echo "$1"
}
check() {
echo "TODO: Implement ${FUNCNAME[0]}."
exit 1
}
install() {
echo "TODO: Implement ${FUNCNAME[0]}."
exit 1
}
# http://ahmed.amayem.com/bash-indirect-expansion-exploration
deps=$2"[*]"
"$1" "${!deps}"

7
environment/os/ubuntu-18.04.sh
View File

@ -18,6 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-dev libxml2-dev # llvm
libedit-dev libpcre3-dev automake bison # swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
@ -27,6 +28,7 @@ TOOLCHAIN_RUN_DEPS=(
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
@ -42,15 +44,20 @@ MEMGRAPH_BUILD_DEPS=(
sbcl # custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
apt install -y $1
}
deps=$2"[*]"
"$1" "${!deps}"

7
environment/os/ubuntu-20.04.sh
View File

@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
libreadline8 # for cmake and llvm
libffi7 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
@ -43,13 +45,17 @@ MEMGRAPH_BUILD_DEPS=(
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
cd "$DIR"
apt update
@ -74,5 +80,6 @@ install() {
apt install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

546
environment/toolchain/v3.sh Executable file
View File

@ -0,0 +1,546 @@
#!/bin/bash -e
# helpers
pushd () { command pushd "$@" > /dev/null; }
popd () { command popd "$@" > /dev/null; }
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CPUS=$( grep -c processor < /proc/cpuinfo )
cd "$DIR"
source "$DIR/../util.sh"
DISTRO="$(operating_system)"
# toolchain version
TOOLCHAIN_VERSION=3
# package versions used
GCC_VERSION=10.2.0
BINUTILS_VERSION=2.35.1
case "$DISTRO" in
centos-7) # because GDB >= 9 does NOT compile with readline6.
GDB_VERSION=8.3
;;
*)
GDB_VERSION=10.1
;;
esac
CMAKE_VERSION=3.18.4
CPPCHECK_VERSION=2.2
LLVM_VERSION=11.0.0
SWIG_VERSION=4.0.2 # used only for LLVM compilation
# Check for the dependencies.
echo "ALL BUILD PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS
echo "ALL RUN PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS
# check installation directory
NAME=toolchain-v$TOOLCHAIN_VERSION
PREFIX=/opt/$NAME
mkdir -p $PREFIX >/dev/null 2>/dev/null || true
if [ ! -d $PREFIX ] || [ ! -w $PREFIX ]; then
echo "Please make sure that the directory '$PREFIX' exists and is writable by the current user!"
echo
echo "If unsure, execute these commands as root:"
echo " mkdir $PREFIX && chown $USER:$USER $PREFIX"
echo
echo "Press <return> when you have created the directory and granted permissions."
# wait for the directory to be created
while true; do
read
if [ ! -d $PREFIX ] || [ ! -w $PREFIX ]; then
echo
echo "You can't continue before you have created the directory and granted permissions!"
echo
echo "Press <return> when you have created the directory and granted permissions."
else
break
fi
done
fi
# create archives directory
mkdir -p archives
# download all archives
pushd archives
if [ ! -f gcc-$GCC_VERSION.tar.gz ]; then
wget https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz
fi
if [ ! -f binutils-$BINUTILS_VERSION.tar.gz ]; then
wget https://ftp.gnu.org/gnu/binutils/binutils-$BINUTILS_VERSION.tar.gz
fi
if [ ! -f gdb-$GDB_VERSION.tar.gz ]; then
wget https://ftp.gnu.org/gnu/gdb/gdb-$GDB_VERSION.tar.gz
fi
if [ ! -f cmake-$CMAKE_VERSION.tar.gz ]; then
wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz
fi
if [ ! -f swig-$SWIG_VERSION.tar.gz ]; then
wget https://github.com/swig/swig/archive/rel-$SWIG_VERSION.tar.gz -O swig-$SWIG_VERSION.tar.gz
fi
if [ ! -f cppcheck-$CPPCHECK_VERSION.tar.gz ]; then
wget https://github.com/danmar/cppcheck/archive/$CPPCHECK_VERSION.tar.gz -O cppcheck-$CPPCHECK_VERSION.tar.gz
fi
if [ ! -f llvm-$LLVM_VERSION.src.tar.xz ]; then
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/lld-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz
fi
if [ ! -f pahole-gdb-master.zip ]; then
wget https://github.com/PhilArmstrong/pahole-gdb/archive/master.zip -O pahole-gdb-master.zip
fi
# verify all archives
# NOTE: Verification can fail if the archive is signed by another developer. I
# haven't added commands to download all developer GnuPG keys because the
# download is very slow. If the verification fails for you, figure out who has
# signed the archive and download their public key instead.
GPG="gpg --homedir .gnupg"
KEYSERVER="hkp://keyserver.ubuntu.com"
mkdir -p .gnupg
chmod 700 .gnupg
# verify gcc
if [ ! -f gcc-$GCC_VERSION.tar.gz.sig ]; then
wget https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz.sig
fi
# list of valid gcc gnupg keys: https://gcc.gnu.org/mirrors.html
$GPG --keyserver $KEYSERVER --recv-keys 0x3AB00996FC26A641
$GPG --verify gcc-$GCC_VERSION.tar.gz.sig gcc-$GCC_VERSION.tar.gz
# verify binutils
if [ ! -f binutils-$BINUTILS_VERSION.tar.gz.sig ]; then
wget https://ftp.gnu.org/gnu/binutils/binutils-$BINUTILS_VERSION.tar.gz.sig
fi
$GPG --keyserver $KEYSERVER --recv-keys 0xDD9E3C4F
$GPG --verify binutils-$BINUTILS_VERSION.tar.gz.sig binutils-$BINUTILS_VERSION.tar.gz
# verify gdb
if [ ! -f gdb-$GDB_VERSION.tar.gz.sig ]; then
wget https://ftp.gnu.org/gnu/gdb/gdb-$GDB_VERSION.tar.gz.sig
fi
$GPG --keyserver $KEYSERVER --recv-keys 0xFF325CF3
$GPG --verify gdb-$GDB_VERSION.tar.gz.sig gdb-$GDB_VERSION.tar.gz
# verify cmake
if [ ! -f cmake-$CMAKE_VERSION-SHA-256.txt ] || [ ! -f cmake-$CMAKE_VERSION-SHA-256.txt.asc ]; then
wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-SHA-256.txt
wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-SHA-256.txt.asc
# Because CentOS 7 doesn't have the `--ignore-missing` flag for `sha256sum`
# we filter out the missing files from the sums here manually.
cat cmake-$CMAKE_VERSION-SHA-256.txt | grep "cmake-$CMAKE_VERSION.tar.gz" > cmake-$CMAKE_VERSION-SHA-256-filtered.txt
fi
$GPG --keyserver $KEYSERVER --recv-keys 0xC6C265324BBEBDC350B513D02D2CEF1034921684
sha256sum -c cmake-$CMAKE_VERSION-SHA-256-filtered.txt
$GPG --verify cmake-$CMAKE_VERSION-SHA-256.txt.asc cmake-$CMAKE_VERSION-SHA-256.txt
# verify llvm, cfe, lld, clang-tools-extra
if [ ! -f llvm-$LLVM_VERSION.src.tar.xz.sig ]; then
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/lld-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz.sig
fi
# list of valid llvm gnupg keys: https://releases.llvm.org/download.html
$GPG --keyserver $KEYSERVER --recv-keys 0x345AD05D
$GPG --verify llvm-$LLVM_VERSION.src.tar.xz.sig llvm-$LLVM_VERSION.src.tar.xz
$GPG --verify clang-$LLVM_VERSION.src.tar.xz.sig clang-$LLVM_VERSION.src.tar.xz
$GPG --verify lld-$LLVM_VERSION.src.tar.xz.sig lld-$LLVM_VERSION.src.tar.xz
$GPG --verify clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig clang-tools-extra-$LLVM_VERSION.src.tar.xz
$GPG --verify compiler-rt-$LLVM_VERSION.src.tar.xz.sig compiler-rt-$LLVM_VERSION.src.tar.xz
popd
# create build directory
mkdir -p build
pushd build
# compile gcc
if [ ! -f $PREFIX/bin/gcc ]; then
if [ -d gcc-$GCC_VERSION ]; then
rm -rf gcc-$GCC_VERSION
fi
tar -xvf ../archives/gcc-$GCC_VERSION.tar.gz
pushd gcc-$GCC_VERSION
./contrib/download_prerequisites
mkdir build && pushd build
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=gcc-8&arch=amd64&ver=8.3.0-6&stamp=1554588545
../configure -v \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--target=x86_64-linux-gnu \
--prefix=$PREFIX \
--disable-multilib \
--with-system-zlib \
--enable-checking=release \
--enable-languages=c,c++,fortran \
--enable-gold=yes \
--enable-ld=yes \
--enable-lto \
--enable-bootstrap \
--disable-vtable-verify \
--disable-werror \
--without-included-gettext \
--enable-threads=posix \
--enable-nls \
--enable-clocale=gnu \
--enable-libstdcxx-debug \
--enable-libstdcxx-time=yes \
--enable-gnu-unique-object \
--enable-libmpx \
--enable-plugin \
--enable-default-pie \
--with-target-system-zlib \
--with-tune=generic \
--without-cuda-driver
#--program-suffix=$( printf "$GCC_VERSION" | cut -d '.' -f 1,2 ) \
make -j$CPUS
# make -k check # run test suite
make install
popd && popd
fi
# activate toolchain
export PATH=$PREFIX/bin:$PATH
export LD_LIBRARY_PATH=$PREFIX/lib64
# compile binutils
if [ ! -f $PREFIX/bin/ld.gold ]; then
if [ -d binutils-$BINUTILS_VERSION ]; then
rm -rf binutils-$BINUTILS_VERSION
fi
tar -xvf ../archives/binutils-$BINUTILS_VERSION.tar.gz
pushd binutils-$BINUTILS_VERSION
mkdir build && pushd build
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=binutils&arch=amd64&ver=2.32-7&stamp=1553247092
env \
CC=gcc \
CXX=g++ \
CFLAGS="-g -O2" \
CXXFLAGS="-g -O2" \
LDFLAGS="" \
../configure \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--prefix=$PREFIX \
--enable-ld=default \
--enable-gold \
--enable-lto \
--enable-plugins \
--enable-shared \
--enable-threads \
--with-system-zlib \
--enable-deterministic-archives \
--disable-compressed-debug-sections \
--enable-new-dtags \
--disable-werror
make -j$CPUS
# make -k check # run test suite
make install
popd && popd
fi
# compile gdb
if [ ! -f $PREFIX/bin/gdb ]; then
if [ -d gdb-$GDB_VERSION ]; then
rm -rf gdb-$GDB_VERSION
fi
tar -xvf ../archives/gdb-$GDB_VERSION.tar.gz
pushd gdb-$GDB_VERSION
mkdir build && pushd build
# https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0
env \
CC=gcc \
CXX=g++ \
CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
LDFLAGS="-Wl,-z,relro" \
PYTHON="" \
../configure \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--prefix=$PREFIX \
--disable-maintainer-mode \
--disable-dependency-tracking \
--disable-silent-rules \
--disable-gdbtk \
--disable-shared \
--without-guile \
--with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
--with-system-readline \
--with-expat \
--with-system-zlib \
--with-lzma \
--with-babeltrace \
--with-intel-pt \
--enable-tui \
--with-python=python3
make -j$CPUS
make install
popd && popd
fi
# install pahole
if [ ! -d $PREFIX/share/pahole-gdb ]; then
unzip ../archives/pahole-gdb-master.zip
mv pahole-gdb-master $PREFIX/share/pahole-gdb
fi
# setup system gdbinit
if [ ! -f $PREFIX/etc/gdb/gdbinit ]; then
mkdir -p $PREFIX/etc/gdb
cat >$PREFIX/etc/gdb/gdbinit <<EOF
# improve formatting
set print pretty on
set print object on
set print static-members on
set print vtbl on
set print demangle on
set demangle-style gnu-v3
set print sevenbit-strings off
# load libstdc++ pretty printers
add-auto-load-scripts-directory $PREFIX/lib64
add-auto-load-safe-path $PREFIX
# load pahole
python
sys.path.insert(0, "$PREFIX/share/pahole-gdb")
import offsets
import pahole
end
EOF
fi
# compile cmake
if [ ! -f $PREFIX/bin/cmake ]; then
if [ -d cmake-$CMAKE_VERSION ]; then
rm -rf cmake-$CMAKE_VERSION
fi
tar -xvf ../archives/cmake-$CMAKE_VERSION.tar.gz
pushd cmake-$CMAKE_VERSION
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=cmake&arch=amd64&ver=3.13.4-1&stamp=1549799837
echo 'set(CMAKE_SKIP_RPATH ON CACHE BOOL "Skip rpath" FORCE)' >> build-flags.cmake
echo 'set(CMAKE_USE_RELATIVE_PATHS ON CACHE BOOL "Use relative paths" FORCE)' >> build-flags.cmake
echo 'set(CMAKE_C_FLAGS "-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2" CACHE STRING "C flags" FORCE)' >> build-flags.cmake
echo 'set(CMAKE_CXX_FLAGS "-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2" CACHE STRING "C++ flags" FORCE)' >> build-flags.cmake
echo 'set(CMAKE_SKIP_BOOTSTRAP_TEST ON CACHE BOOL "Skip BootstrapTest" FORCE)' >> build-flags.cmake
echo 'set(BUILD_CursesDialog ON CACHE BOOL "Build curses GUI" FORCE)' >> build-flags.cmake
mkdir build && pushd build
../bootstrap \
--prefix=$PREFIX \
--init=../build-flags.cmake \
--parallel=$CPUS \
--system-curl
make -j$CPUS
# make test # run test suite
make install
popd && popd
fi
# compile cppcheck
if [ ! -f $PREFIX/bin/cppcheck ]; then
if [ -d cppcheck-$CPPCHECK_VERSION ]; then
rm -rf cppcheck-$CPPCHECK_VERSION
fi
tar -xvf ../archives/cppcheck-$CPPCHECK_VERSION.tar.gz
pushd cppcheck-$CPPCHECK_VERSION
env \
CC=gcc \
CXX=g++ \
PREFIX=$PREFIX \
FILESDIR=$PREFIX/share/cppcheck \
CFGDIR=$PREFIX/share/cppcheck/cfg \
make -j$CPUS
env \
CC=gcc \
CXX=g++ \
PREFIX=$PREFIX \
FILESDIR=$PREFIX/share/cppcheck \
CFGDIR=$PREFIX/share/cppcheck/cfg \
make install
popd
fi
# compile swig
if [ ! -d swig-$SWIG_VERSION/install ]; then
if [ -d swig-$SWIG_VERSION ]; then
rm -rf swig-$SWIG_VERSION
fi
tar -xvf ../archives/swig-$SWIG_VERSION.tar.gz
mv swig-rel-$SWIG_VERSION swig-$SWIG_VERSION
pushd swig-$SWIG_VERSION
./autogen.sh
mkdir build && pushd build
../configure --prefix=$DIR/build/swig-$SWIG_VERSION/install
make -j$CPUS
make install
popd && popd
fi
# compile llvm
if [ ! -f $PREFIX/bin/clang ]; then
if [ -d llvm-$LLVM_VERSION ]; then
rm -rf llvm-$LLVM_VERSION
fi
tar -xvf ../archives/llvm-$LLVM_VERSION.src.tar.xz
mv llvm-$LLVM_VERSION.src llvm-$LLVM_VERSION
tar -xvf ../archives/clang-$LLVM_VERSION.src.tar.xz
mv clang-$LLVM_VERSION.src llvm-$LLVM_VERSION/tools/clang
tar -xvf ../archives/lld-$LLVM_VERSION.src.tar.xz
mv lld-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/tools/lld
tar -xvf ../archives/clang-tools-extra-$LLVM_VERSION.src.tar.xz
mv clang-tools-extra-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/tools/clang/tools/extra
tar -xvf ../archives/compiler-rt-$LLVM_VERSION.src.tar.xz
mv compiler-rt-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/projects/compiler-rt
pushd llvm-$LLVM_VERSION
mkdir build && pushd build
# activate swig
export PATH=$DIR/build/swig-$SWIG_VERSION/install/bin:$PATH
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=llvm-toolchain-7&arch=amd64&ver=1%3A7.0.1%7E%2Brc2-1%7Eexp1&stamp=1541506173&raw=0
cmake .. \
-DCMAKE_C_COMPILER=$PREFIX/bin/gcc \
-DCMAKE_CXX_COMPILER=$PREFIX/bin/g++ \
-DCMAKE_CXX_LINK_FLAGS="-L$PREFIX/lib64 -Wl,-rpath,$PREFIX/lib64" \
-DCMAKE_INSTALL_PREFIX=$PREFIX \
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DCMAKE_CXX_FLAGS_RELWITHDEBINFO="-O2 -DNDEBUG" \
-DCMAKE_CXX_FLAGS=' -fuse-ld=gold -fPIC -Wno-unused-command-line-argument -Wno-unknown-warning-option' \
-DCMAKE_C_FLAGS=' -fuse-ld=gold -fPIC -Wno-unused-command-line-argument -Wno-unknown-warning-option' \
-DLLVM_LINK_LLVM_DYLIB=ON \
-DLLVM_INSTALL_UTILS=ON \
-DLLVM_VERSION_SUFFIX= \
-DLLVM_BUILD_LLVM_DYLIB=ON \
-DLLVM_ENABLE_RTTI=ON \
-DLLVM_ENABLE_FFI=ON \
-DLLVM_BINUTILS_INCDIR=$PREFIX/include/ \
-DLLVM_USE_PERF=yes
make -j$CPUS
make -j$CPUS check-clang # run clang test suite
make -j$CPUS check-lld # run lld test suite
make install
popd && popd
fi
# create README
if [ ! -f $PREFIX/README.md ]; then
cat >$PREFIX/README.md <<EOF
# Memgraph Toolchain v$TOOLCHAIN_VERSION
## Included tools
- GCC $GCC_VERSION
- Binutils $BINUTILS_VERSION
- GDB $GDB_VERSION
- CMake $CMAKE_VERSION
- Cppcheck $CPPCHECK_VERSION
- LLVM (Clang, LLD, compiler-rt, Clang tools extra) $LLVM_VERSION
## Required libraries
In order to be able to run all of these tools you should install the following
packages:
\`\`\`
$($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)
\`\`\`
## Usage
In order to use the toolchain you just have to source the activation script:
\`\`\`
source $PREFIX/activate
\`\`\`
EOF
fi
# create activation script
if [ ! -f $PREFIX/activate ]; then
cat >$PREFIX/activate <<EOF
# This file must be used with "source $PREFIX/activate" *from bash*
# You can't run it directly!
env_error="You already have an active virtual environment!"
# zsh does not recognize the option -t of the command type
# therefore we use the alternative whence -w
if [[ "\$ZSH_NAME" == "zsh" ]]; then
# check for active virtual environments
if [ "\$( whence -w deactivate )" != "deactivate: none" ]; then
echo \$env_error
return 0;
fi
# any other shell
else
# check for active virtual environments
if [ "\$( type -t deactivate )" != "" ]; then
echo \$env_error
return 0
fi
fi
# check that we aren't root
if [[ "\$USER" == "root" ]]; then
echo "You shouldn't use the toolchain as root!"
return 0
fi
# save original environment
export ORIG_PATH=\$PATH
export ORIG_PS1=\$PS1
export ORIG_LD_LIBRARY_PATH=\$LD_LIBRARY_PATH
# activate new environment
export PATH=$PREFIX/bin:\$PATH
export PS1="($NAME) \$PS1"
export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64
# disable root
function su () {
echo "You don't want to use root functions while using the toolchain!"
return 1
}
function sudo () {
echo "You don't want to use root functions while using the toolchain!"
return 1
}
# create deactivation function
function deactivate() {
export PATH=\$ORIG_PATH
export PS1=\$ORIG_PS1
export LD_LIBRARY_PATH=\$ORIG_LD_LIBRARY_PATH
unset ORIG_PATH ORIG_PS1 ORIG_LD_LIBRARY_PATH
unset -f su sudo deactivate
}
EOF
fi
# create toolchain archive
if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then
tar --owner=root --group=root -cpvzf $NAME-binaries-$DISTRO.tar.gz -C /opt $NAME
fi
# output final instructions
echo -e "\n\n"
echo "All tools have been built. They are installed in '$PREFIX'."
echo "In order to distribute the tools to someone else, an archive with the toolchain was created in the 'build' directory."
echo "If you want to install the packed tools you should execute the following command:"
echo
echo " tar -xvzf build/$NAME-binaries.tar.gz -C /opt"
echo
echo "Because the tools were built on this machine, you should probably change the permissions of the installation directory using:"
echo
echo " OPTIONAL: chown -R root:root $PREFIX"
echo
echo "In order to use all of the newly compiled tools you should use the prepared activation script:"
echo
echo " source $PREFIX/activate"
echo
echo "Or, for more advanced uses, you can add the following lines to your script:"
echo
echo " export PATH=$PREFIX/bin:\$PATH"
echo " export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64"
echo
echo "Enjoy!"

133
include/mg_procedure.h
View File

@ -17,10 +17,11 @@ extern "C" {
/// addition to efficiency, Memgraph can set the limit on allowed allocations
/// thus providing some safety with regards to memory usage. The allocated
/// memory is only valid during the execution of mgp_main. You must not allocate
/// global resources with these functions. None of the functions are
/// global resources with these functions and none of the functions are
/// thread-safe, because we provide a single thread of execution when invoking a
/// custom procedure. This allows Memgraph to be more efficient as stated
/// before.
/// custom procedure. For allocating global resources, you can use the _global
/// variations of the aforementioned allocators. This allows Memgraph to be
/// more efficient as explained before.
///@{
/// Provides memory managament access and state.
@ -39,8 +40,7 @@ void *mgp_alloc(struct mgp_memory *memory, size_t size_in_bytes);
/// `alignment` must be a power of 2 value.
/// The returned pointer must be freed with mgp_free.
/// NULL is returned if unable to serve the requested allocation.
void *mgp_aligned_alloc(struct mgp_memory *memory, size_t size_in_bytes,
size_t alignment);
void *mgp_aligned_alloc(struct mgp_memory *memory, size_t size_in_bytes, size_t alignment);
/// Deallocate an allocation from mgp_alloc or mgp_aligned_alloc.
/// Unlike free, this function is not thread-safe.
@ -48,6 +48,26 @@ void *mgp_aligned_alloc(struct mgp_memory *memory, size_t size_in_bytes,
/// The behavior is undefined if `ptr` is not a value returned from a prior
/// mgp_alloc or mgp_aligned_alloc call with the corresponding `memory`.
void mgp_free(struct mgp_memory *memory, void *ptr);
/// Allocate a global block of memory with given size in bytes.
/// This function can be used to allocate global memory that persists
/// beyond a single invocation of mgp_main.
/// The returned pointer must be freed with mgp_global_free.
/// NULL is returned if unable to serve the requested allocation.
void *mgp_global_alloc(size_t size_in_bytes);
/// Allocate an aligned global block of memory with given size in bytes.
/// This function can be used to allocate global memory that persists
/// beyond a single invocation of mgp_main.
/// The returned pointer must be freed with mgp_global_free.
/// NULL is returned if unable to serve the requested allocation.
void *mgp_global_aligned_alloc(size_t size_in_bytes, size_t alignment);
/// Deallocate an allocation from mgp_global_alloc or mgp_global_aligned_alloc.
/// If `ptr` is NULL, this function does nothing.
/// The behavior is undefined if `ptr` is not a value returned from a prior
/// mgp_global_alloc() or mgp_global_aligned_alloc().
void mgp_global_free(void *p);
///@}
/// @name Operations on mgp_value
@ -119,8 +139,7 @@ struct mgp_value *mgp_value_make_double(double val, struct mgp_memory *memory);
/// Construct a character string value from a NULL terminated string.
/// You need to free the instance through mgp_value_destroy.
/// NULL is returned if unable to allocate a mgp_value.
struct mgp_value *mgp_value_make_string(const char *val,
struct mgp_memory *memory);
struct mgp_value *mgp_value_make_string(const char *val, struct mgp_memory *memory);
/// Create a mgp_value storing a mgp_list.
/// You need to free the instance through mgp_value_destroy. The ownership of
@ -238,8 +257,7 @@ const struct mgp_path *mgp_value_get_path(const struct mgp_value *val);
/// of mgp_value, but it will not contain any elements. Therefore,
/// mgp_list_size will return 0.
/// NULL is returned if unable to allocate a new list.
struct mgp_list *mgp_list_make_empty(size_t capacity,
struct mgp_memory *memory);
struct mgp_list *mgp_list_make_empty(size_t capacity, struct mgp_memory *memory);
/// Free the memory used by the given mgp_list and contained elements.
void mgp_list_destroy(struct mgp_list *list);
@ -288,8 +306,7 @@ void mgp_map_destroy(struct mgp_map *map);
/// you still need to free their memory explicitly.
/// Return non-zero on success, or 0 if there's no memory to insert a new
/// mapping or a previous mapping already exists.
int mgp_map_insert(struct mgp_map *map, const char *key,
const struct mgp_value *value);
int mgp_map_insert(struct mgp_map *map, const char *key, const struct mgp_value *value);
/// Return the number of items stored in mgp_map.
size_t mgp_map_size(const struct mgp_map *map);
@ -314,8 +331,7 @@ struct mgp_map_items_iterator;
/// The returned mgp_map_items_iterator needs to be deallocated with
/// mgp_map_items_iterator_destroy.
/// NULL is returned if unable to allocate a new iterator.
struct mgp_map_items_iterator *mgp_map_iter_items(const struct mgp_map *map,
struct mgp_memory *memory);
struct mgp_map_items_iterator *mgp_map_iter_items(const struct mgp_map *map, struct mgp_memory *memory);
/// Deallocate memory used by mgp_map_items_iterator.
void mgp_map_items_iterator_destroy(struct mgp_map_items_iterator *it);
@ -328,27 +344,23 @@ void mgp_map_items_iterator_destroy(struct mgp_map_items_iterator *it);
/// as the value before, and use them after invoking
/// mgp_map_items_iterator_next.
/// NULL is returned if the end of the iteration has been reached.
const struct mgp_map_item *mgp_map_items_iterator_get(
const struct mgp_map_items_iterator *it);
const struct mgp_map_item *mgp_map_items_iterator_get(const struct mgp_map_items_iterator *it);
/// Advance the iterator to the next item stored in map and return it.
/// The previous pointer obtained through mgp_map_items_iterator_get will
/// be invalidated, but the pointers to key and value will remain valid.
/// NULL is returned if the end of the iteration has been reached.
const struct mgp_map_item *mgp_map_items_iterator_next(
struct mgp_map_items_iterator *it);
const struct mgp_map_item *mgp_map_items_iterator_next(struct mgp_map_items_iterator *it);
/// Create a path with the copy of the given starting vertex.
/// You need to free the created instance with mgp_path_destroy.
/// NULL is returned if unable to allocate a path.
struct mgp_path *mgp_path_make_with_start(const struct mgp_vertex *vertex,
struct mgp_memory *memory);
struct mgp_path *mgp_path_make_with_start(const struct mgp_vertex *vertex, struct mgp_memory *memory);
/// Copy a mgp_path.
/// Returned pointer must be freed with mgp_path_destroy.
/// NULL is returned if unable to allocate a mgp_path.
struct mgp_path *mgp_path_copy(const struct mgp_path *path,
struct mgp_memory *memory);
struct mgp_path *mgp_path_copy(const struct mgp_path *path, struct mgp_memory *memory);
/// Free the memory used by the given mgp_path and contained vertices and edges.
void mgp_path_destroy(struct mgp_path *path);
@ -370,14 +382,12 @@ size_t mgp_path_size(const struct mgp_path *path);
/// Return the vertex from a path at given index.
/// The valid index range is [0, mgp_path_size].
/// NULL is returned if index is out of range.
const struct mgp_vertex *mgp_path_vertex_at(const struct mgp_path *path,
size_t index);
const struct mgp_vertex *mgp_path_vertex_at(const struct mgp_path *path, size_t index);
/// Return the edge from a path at given index.
/// The valid index range is [0, mgp_path_size - 1].
/// NULL is returned if index is out of range.
const struct mgp_edge *mgp_path_edge_at(const struct mgp_path *path,
size_t index);
const struct mgp_edge *mgp_path_edge_at(const struct mgp_path *path, size_t index);
/// Return non-zero if given paths are equal, otherwise 0.
int mgp_path_equal(const struct mgp_path *p1, const struct mgp_path *p2);
@ -408,9 +418,7 @@ struct mgp_result_record *mgp_result_new_record(struct mgp_result *res);
/// Return 0 if there's no memory to copy the mgp_value to mgp_result_record or
/// if the combination of `field_name` and `val` does not satisfy the
/// procedure's result signature.
int mgp_result_record_insert(struct mgp_result_record *record,
const char *field_name,
const struct mgp_value *val);
int mgp_result_record_insert(struct mgp_result_record *record, const char *field_name, const struct mgp_value *val);
///@}
/// @name Graph Constructs
@ -446,15 +454,13 @@ struct mgp_property {
/// When the mgp_properties_iterator_next is invoked, the previous
/// mgp_property is invalidated and its value must not be used.
/// NULL is returned if the end of the iteration has been reached.
const struct mgp_property *mgp_properties_iterator_get(
const struct mgp_properties_iterator *it);
const struct mgp_property *mgp_properties_iterator_get(const struct mgp_properties_iterator *it);
/// Advance the iterator to the next property and return it.
/// The previous mgp_property obtained through mgp_properties_iterator_get
/// will be invalidated, and you must not use its value.
/// NULL is returned if the end of the iteration has been reached.
const struct mgp_property *mgp_properties_iterator_next(
struct mgp_properties_iterator *it);
const struct mgp_property *mgp_properties_iterator_next(struct mgp_properties_iterator *it);
/// Iterator over edges of a vertex.
struct mgp_edges_iterator;
@ -475,8 +481,7 @@ struct mgp_vertex_id mgp_vertex_get_id(const struct mgp_vertex *v);
/// Copy a mgp_vertex.
/// Returned pointer must be freed with mgp_vertex_destroy.
/// NULL is returned if unable to allocate a mgp_vertex.
struct mgp_vertex *mgp_vertex_copy(const struct mgp_vertex *v,
struct mgp_memory *memory);
struct mgp_vertex *mgp_vertex_copy(const struct mgp_vertex *v, struct mgp_memory *memory);
/// Free the memory used by a mgp_vertex.
void mgp_vertex_destroy(struct mgp_vertex *v);
@ -495,43 +500,37 @@ struct mgp_label mgp_vertex_label_at(const struct mgp_vertex *v, size_t index);
int mgp_vertex_has_label(const struct mgp_vertex *v, struct mgp_label label);
/// Return non-zero if the given vertex has a label with given name.
int mgp_vertex_has_label_named(const struct mgp_vertex *v,
const char *label_name);
int mgp_vertex_has_label_named(const struct mgp_vertex *v, const char *label_name);
/// Get a copy of a vertex property mapped to a given name.
/// Returned value must be freed with mgp_value_destroy.
/// NULL is returned if unable to allocate a mgp_value.
struct mgp_value *mgp_vertex_get_property(const struct mgp_vertex *v,
const char *property_name,
struct mgp_value *mgp_vertex_get_property(const struct mgp_vertex *v, const char *property_name,
struct mgp_memory *memory);
/// Start iterating over properties stored in the given vertex.
/// The returned mgp_properties_iterator needs to be deallocated with
/// mgp_properties_iterator_destroy.
/// NULL is returned if unable to allocate a new iterator.
struct mgp_properties_iterator *mgp_vertex_iter_properties(
const struct mgp_vertex *v, struct mgp_memory *memory);
struct mgp_properties_iterator *mgp_vertex_iter_properties(const struct mgp_vertex *v, struct mgp_memory *memory);
/// Start iterating over inbound edges of the given vertex.
/// The returned mgp_edges_iterator needs to be deallocated with
/// mgp_edges_iterator_destroy.
/// NULL is returned if unable to allocate a new iterator.
struct mgp_edges_iterator *mgp_vertex_iter_in_edges(const struct mgp_vertex *v,
struct mgp_memory *memory);
struct mgp_edges_iterator *mgp_vertex_iter_in_edges(const struct mgp_vertex *v, struct mgp_memory *memory);
/// Start iterating over outbound edges of the given vertex.
/// The returned mgp_edges_iterator needs to be deallocated with
/// mgp_edges_iterator_destroy.
/// NULL is returned if unable to allocate a new iterator.
struct mgp_edges_iterator *mgp_vertex_iter_out_edges(const struct mgp_vertex *v,
struct mgp_memory *memory);
struct mgp_edges_iterator *mgp_vertex_iter_out_edges(const struct mgp_vertex *v, struct mgp_memory *memory);
/// Get the current edge pointed to by the iterator.
/// When the mgp_edges_iterator_next is invoked, the previous
/// mgp_edge is invalidated and its value must not be used.
/// NULL is returned if the end of the iteration has been reached.
const struct mgp_edge *mgp_edges_iterator_get(
const struct mgp_edges_iterator *it);
const struct mgp_edge *mgp_edges_iterator_get(const struct mgp_edges_iterator *it);
/// Advance the iterator to the next edge and return it.
/// The previous mgp_edge obtained through mgp_edges_iterator_get
@ -552,8 +551,7 @@ struct mgp_edge_id mgp_edge_get_id(const struct mgp_edge *e);
/// Copy a mgp_edge.
/// Returned pointer must be freed with mgp_edge_destroy.
/// NULL is returned if unable to allocate a mgp_edge.
struct mgp_edge *mgp_edge_copy(const struct mgp_edge *e,
struct mgp_memory *memory);
struct mgp_edge *mgp_edge_copy(const struct mgp_edge *e, struct mgp_memory *memory);
/// Free the memory used by a mgp_edge.
void mgp_edge_destroy(struct mgp_edge *e);
@ -573,16 +571,13 @@ const struct mgp_vertex *mgp_edge_get_to(const struct mgp_edge *e);
/// Get a copy of a edge property mapped to a given name.
/// Returned value must be freed with mgp_value_destroy.
/// NULL is returned if unable to allocate a mgp_value.
struct mgp_value *mgp_edge_get_property(const struct mgp_edge *e,
const char *property_name,
struct mgp_memory *memory);
struct mgp_value *mgp_edge_get_property(const struct mgp_edge *e, const char *property_name, struct mgp_memory *memory);
/// Start iterating over properties stored in the given edge.
/// The returned mgp_properties_iterator needs to be deallocated with
/// mgp_properties_iterator_destroy.
/// NULL is returned if unable to allocate a new iterator.
struct mgp_properties_iterator *mgp_edge_iter_properties(
const struct mgp_edge *e, struct mgp_memory *memory);
struct mgp_properties_iterator *mgp_edge_iter_properties(const struct mgp_edge *e, struct mgp_memory *memory);
/// State of the graph database.
struct mgp_graph;
@ -590,8 +585,7 @@ struct mgp_graph;
/// Return the vertex corresponding to given ID.
/// The returned vertex must be freed using mgp_vertex_destroy.
/// NULL is returned if unable to allocate the vertex or if ID is not valid.
struct mgp_vertex *mgp_graph_get_vertex_by_id(const struct mgp_graph *g,
struct mgp_vertex_id id,
struct mgp_vertex *mgp_graph_get_vertex_by_id(const struct mgp_graph *g, struct mgp_vertex_id id,
struct mgp_memory *memory);
/// Iterator over vertices.
@ -604,22 +598,19 @@ void mgp_vertices_iterator_destroy(struct mgp_vertices_iterator *it);
/// The returned mgp_vertices_iterator needs to be deallocated with
/// mgp_vertices_iterator_destroy.
/// NULL is returned if unable to allocate a new iterator.
struct mgp_vertices_iterator *mgp_graph_iter_vertices(
const struct mgp_graph *g, struct mgp_memory *memory);
struct mgp_vertices_iterator *mgp_graph_iter_vertices(const struct mgp_graph *g, struct mgp_memory *memory);
/// Get the current vertex pointed to by the iterator.
/// When the mgp_vertices_iterator_next is invoked, the previous
/// mgp_vertex is invalidated and its value must not be used.
/// NULL is returned if the end of the iteration has been reached.
const struct mgp_vertex *mgp_vertices_iterator_get(
const struct mgp_vertices_iterator *it);
const struct mgp_vertex *mgp_vertices_iterator_get(const struct mgp_vertices_iterator *it);
/// Advance the iterator to the next vertex and return it.
/// The previous mgp_vertex obtained through mgp_vertices_iterator_get
/// will be invalidated, and you must not use its value.
/// NULL is returned if the end of the iteration has been reached.
const struct mgp_vertex *mgp_vertices_iterator_next(
struct mgp_vertices_iterator *it);
const struct mgp_vertex *mgp_vertices_iterator_next(struct mgp_vertices_iterator *it);
///@}
/// @name Type System
@ -718,8 +709,8 @@ struct mgp_proc;
/// Passed in arguments will not live longer than the callback's execution.
/// Therefore, you must not store them globally or use the passed in mgp_memory
/// to allocate global resources.
typedef void (*mgp_proc_cb)(const struct mgp_list *, const struct mgp_graph *,
struct mgp_result *, struct mgp_memory *);
typedef void (*mgp_proc_cb)(const struct mgp_list *, const struct mgp_graph *, struct mgp_result *,
struct mgp_memory *);
/// Register a read-only procedure with a module.
///
@ -730,9 +721,7 @@ typedef void (*mgp_proc_cb)(const struct mgp_list *, const struct mgp_graph *,
///
/// NULL is returned if unable to allocate memory for mgp_proc; if `name` is
/// not valid or a procedure with the same name was already registered.
struct mgp_proc *mgp_module_add_read_procedure(struct mgp_module *module,
const char *name,
mgp_proc_cb cb);
struct mgp_proc *mgp_module_add_read_procedure(struct mgp_module *module, const char *name, mgp_proc_cb cb);
/// Add a required argument to a procedure.
///
@ -748,8 +737,7 @@ struct mgp_proc *mgp_module_add_read_procedure(struct mgp_module *module,
/// 0 is returned if unable to allocate memory for an argument; if invoking this
/// function after setting an optional argument or if `name` is not valid.
/// Non-zero is returned on success.
int mgp_proc_add_arg(struct mgp_proc *proc, const char *name,
const struct mgp_type *type);
int mgp_proc_add_arg(struct mgp_proc *proc, const char *name, const struct mgp_type *type);
/// Add an optional argument with a default value to a procedure.
///
@ -772,8 +760,7 @@ int mgp_proc_add_arg(struct mgp_proc *proc, const char *name,
/// 0 is returned if unable to allocate memory for an argument; if `name` is
/// not valid or `default_value` does not satisfy `type`. Non-zero is returned
/// on success.
int mgp_proc_add_opt_arg(struct mgp_proc *proc, const char *name,
const struct mgp_type *type,
int mgp_proc_add_opt_arg(struct mgp_proc *proc, const char *name, const struct mgp_type *type,
const struct mgp_value *default_value);
/// Add a result field to a procedure.
@ -787,15 +774,13 @@ int mgp_proc_add_opt_arg(struct mgp_proc *proc, const char *name,
/// 0 is returned if unable to allocate memory for a result field; if
/// `name` is not valid or if a result field with the same name was already
/// added. Non-zero is returned on success.
int mgp_proc_add_result(struct mgp_proc *proc, const char *name,
const struct mgp_type *type);
int mgp_proc_add_result(struct mgp_proc *proc, const char *name, const struct mgp_type *type);
/// Add a result field to a procedure and mark it as deprecated.
///
/// This is the same as mgp_proc_add_result, but the result field will be marked
/// as deprecated.
int mgp_proc_add_deprecated_result(struct mgp_proc *proc, const char *name,
const struct mgp_type *type);
int mgp_proc_add_deprecated_result(struct mgp_proc *proc, const char *name, const struct mgp_type *type);
///@}
/// @name Execution

17
include/mgp.py
View File

@ -627,8 +627,11 @@ def _typing_to_cypher_type(type_):
if complex_type == typing.Union:
# If we have a Union with NoneType inside, it means we are building
# a nullable type.
if isinstance(None, type_args):
types = tuple(t for t in type_args if not isinstance(None, t))
# isinstance doesn't work here because subscripted generics cannot
# be used with class and instance checks. type comparison should be
# fine because subclasses are not used.
if type(None) in type_args:
types = tuple(t for t in type_args if t is not type(None)) # noqa E721
if len(types) == 1:
type_arg, = types
else:
@ -683,7 +686,15 @@ def _typing_to_cypher_type(type_):
return _mgp.type_nullable(simple_type)
return _mgp.type_nullable(parse_typing(type_arg_as_str))
elif type_as_str.startswith('typing.List'):
type_arg_as_str, = parse_type_args(type_as_str)
type_arg_as_str = parse_type_args(type_as_str)
if len(type_arg_as_str) > 1:
# Nested object could be a type consisting of a list of types (e.g. mgp.Map)
# so we need to join the parts.
type_arg_as_str = ', '.join(type_arg_as_str)
else:
type_arg_as_str = type_arg_as_str[0]
simple_type = get_simple_type(type_arg_as_str)
if simple_type is not None:
return _mgp.type_list(simple_type)

1
libs/.gitignore vendored
View File

@ -4,3 +4,4 @@
!cleanup.sh
!CMakeLists.txt
!__main.cpp
!jemalloc.cmake

18
libs/CMakeLists.txt
View File

@ -8,6 +8,8 @@ if (NPROC EQUAL 0)
set(NPROC 1)
endif()
set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR})
# convenience functions
function(import_header_library name include_dir)
add_library(${name} INTERFACE IMPORTED GLOBAL)
@ -89,11 +91,9 @@ import_external_library(antlr4 STATIC
CMAKE_ARGS # http://stackoverflow.com/questions/37096062/get-a-basic-c-program-to-compile-using-clang-on-ubuntu-16/38385967#38385967
-DWITH_LIBCXX=OFF # because of debian bug
-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
-DCMAKE_CXX_STANDARD=20
BUILD_COMMAND $(MAKE) antlr4_static
# Make a License.txt out of thin air, so that antlr4.6 knows how to build.
# When we upgrade antlr, this will no longer be needed.
INSTALL_COMMAND touch ${CMAKE_CURRENT_SOURCE_DIR}/antlr4/runtime/Cpp/License.txt
COMMAND $(MAKE) install)
INSTALL_COMMAND $(MAKE) install)
# Setup google benchmark.
import_external_library(benchmark STATIC
@ -207,8 +207,18 @@ import_external_library(mgclient STATIC
find_package(OpenSSL REQUIRED)
target_link_libraries(mgclient INTERFACE ${OPENSSL_LIBRARIES})
add_external_project(mgconsole
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/mgconsole
CMAKE_ARGS
-DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_BINARY_DIR}
BUILD_COMMAND $(MAKE) mgconsole)
add_custom_target(mgconsole DEPENDS mgconsole-proj)
# Setup spdlog
import_external_library(spdlog STATIC
${CMAKE_CURRENT_SOURCE_DIR}/spdlog/${CMAKE_INSTALL_LIBDIR}/libspdlog.a
${CMAKE_CURRENT_SOURCE_DIR}/spdlog/include
BUILD_COMMAND $(MAKE) spdlog)
include(jemalloc.cmake)

55
libs/jemalloc.cmake Normal file
View File

@ -0,0 +1,55 @@
set(JEMALLOC_DIR "${LIB_DIR}/jemalloc")
set(JEMALLOC_SRCS
${JEMALLOC_DIR}/src/arena.c
${JEMALLOC_DIR}/src/background_thread.c
${JEMALLOC_DIR}/src/base.c
${JEMALLOC_DIR}/src/bin.c
${JEMALLOC_DIR}/src/bitmap.c
${JEMALLOC_DIR}/src/ckh.c
${JEMALLOC_DIR}/src/ctl.c
${JEMALLOC_DIR}/src/div.c
${JEMALLOC_DIR}/src/extent.c
${JEMALLOC_DIR}/src/extent_dss.c
${JEMALLOC_DIR}/src/extent_mmap.c
${JEMALLOC_DIR}/src/hash.c
${JEMALLOC_DIR}/src/hook.c
${JEMALLOC_DIR}/src/jemalloc.c
${JEMALLOC_DIR}/src/large.c
${JEMALLOC_DIR}/src/log.c
${JEMALLOC_DIR}/src/malloc_io.c
${JEMALLOC_DIR}/src/mutex.c
${JEMALLOC_DIR}/src/mutex_pool.c
${JEMALLOC_DIR}/src/nstime.c
${JEMALLOC_DIR}/src/pages.c
${JEMALLOC_DIR}/src/prng.c
${JEMALLOC_DIR}/src/prof.c
${JEMALLOC_DIR}/src/rtree.c
${JEMALLOC_DIR}/src/sc.c
${JEMALLOC_DIR}/src/stats.c
${JEMALLOC_DIR}/src/sz.c
${JEMALLOC_DIR}/src/tcache.c
${JEMALLOC_DIR}/src/test_hooks.c
${JEMALLOC_DIR}/src/ticker.c
${JEMALLOC_DIR}/src/tsd.c
${JEMALLOC_DIR}/src/witness.c
${JEMALLOC_DIR}/src/safety_check.c
)
add_library(jemalloc ${JEMALLOC_SRCS})
target_include_directories(jemalloc PUBLIC "${JEMALLOC_DIR}/include")
find_package(Threads REQUIRED)
target_link_libraries(jemalloc PUBLIC Threads::Threads)
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1)
endif()
target_compile_options(jemalloc PRIVATE -Wno-redundant-decls)
# for RTLD_NEXT
target_compile_definitions(jemalloc PRIVATE _GNU_SOURCE)
set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1)

177
libs/setup.sh
View File

@ -2,8 +2,9 @@
# Download external dependencies.
local_cache_host=${MGDEPS_CACHE_HOST_PORT:-mgdeps-cache:8000}
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd ${working_dir}
cd "${working_dir}"
# Clones a git repository and optionally cherry picks additional commits. The
# function will try to preserve any local changes in the repo.
@ -15,7 +16,11 @@ clone () {
shift 3
# Clone if there's no repo.
if [[ ! -d "$dir_name" ]]; then
git clone "$git_repo" "$dir_name"
echo "Cloning from $git_repo"
# If the clone fails, it doesn't make sense to continue with the function
# execution but the whole script should continue executing because we might
# clone the same repo from a different source.
git clone "$git_repo" "$dir_name" || return 1
fi
pushd "$dir_name"
# Just fetch new commits from remote repository. Don't merge/pull them in, so
@ -29,12 +34,17 @@ clone () {
# Stash regardless of local_changes, so that a user gets a message on stdout.
git stash
# Checkout the primary commit (there's no need to pull/merge).
git checkout $checkout_id
# The checkout fail should exit this script immediately because the target
# commit is not there and that will most likely create build-time errors.
git checkout "$checkout_id" || exit 1
# Apply any optional cherry pick fixes.
while [[ $# -ne 0 ]]; do
local cherry_pick_id=$1
shift
git cherry-pick -n $cherry_pick_id
# The cherry-pick fail should exit this script immediately because the
# target commit is not there and that will most likely create build-time
# errors.
git cherry-pick -n "$cherry_pick_id" || exit 1
done
# Reapply any local changes.
if [[ $local_changes == true ]]; then
@ -43,81 +53,188 @@ clone () {
popd
}
file_get_try_double () {
primary_url="$1"
secondary_url="$2"
echo "Download primary from $primary_url secondary from $secondary_url"
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
filename="$(basename "$secondary_url")"
wget -nv "$primary_url" -O "$filename" || wget -nv "$secondary_url" -O "$filename" || exit 1
echo ""
}
repo_clone_try_double () {
primary_url="$1"
secondary_url="$2"
folder_name="$3"
ref="$4"
echo "Cloning primary from $primary_url secondary from $secondary_url"
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
clone "$primary_url" "$folder_name" "$ref" || clone "$secondary_url" "$folder_name" "$ref" || exit 1
echo ""
}
# List all dependencies.
# The reason for introducing primary and secondary urls are:
# * HTTPS is hard to cache
# * Remote development workflow is more flexible if people don't have to connect to VPN
# * Direct download from the "source of truth" is slower and unreliable because of the whole internet in-between
# * When a new dependency has to be added, both urls could be the same, later someone could optimize if required
# The goal of having primary urls is to have links to the "local" cache of
# dependencies where these dependencies could be downloaded as fast as
# possible. The actual cache server could be on your local machine, on a
# dedicated machine inside the build cluster or on the actual build machine.
# Download from primary_urls might fail because the cache is not installed.
declare -A primary_urls=(
["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.9.2-complete.jar"
["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
["fmt"]="http://$local_cache_host/git/fmt.git"
["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
["gtest"]="http://$local_cache_host/git/googletest.git"
["gflags"]="http://$local_cache_host/git/gflags.git"
["libbcrypt"]="http://$local_cache_host/git/libbcrypt.git"
["bzip2"]="http://$local_cache_host/git/bzip2.git"
["zlib"]="http://$local_cache_host/git/zlib.git"
["rocksdb"]="http://$local_cache_host/git/rocksdb.git"
["mgclient"]="http://$local_cache_host/git/mgclient.git"
["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
["mgconsole"]="http://$local_cache_host/git/mgconsole.git"
["spdlog"]="http://$local_cache_host/git/spdlog"
["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
)
# The goal of secondary urls is to have links to the "source of truth" of
# dependencies, e.g., Github or S3. Download from secondary urls, if happens
# at all, should never fail. In other words, if it fails, the whole build
# should fail.
declare -A secondary_urls=(
["antlr4-code"]="https://github.com/antlr/antlr4.git"
["antlr4-generator"]="http://www.antlr.org/download/antlr-4.9.2-complete.jar"
["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
["fmt"]="https://github.com/fmtlib/fmt.git"
["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
["gbenchmark"]="https://github.com/google/benchmark.git"
["gtest"]="https://github.com/google/googletest.git"
["gflags"]="https://github.com/memgraph/gflags.git"
["libbcrypt"]="https://github.com/rg3/libbcrypt"
["bzip2"]="https://github.com/VFR-maniac/bzip2"
["zlib"]="https://github.com/madler/zlib.git"
["rocksdb"]="https://github.com/facebook/rocksdb.git"
["mgclient"]="https://github.com/memgraph/mgclient.git"
["pymgclient"]="https://github.com/memgraph/pymgclient.git"
["mgconsole"]="http://github.com/memgraph/mgconsole.git"
["spdlog"]="https://github.com/gabime/spdlog"
["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
)
# antlr
antlr_generator_filename="antlr-4.6-complete.jar"
# wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename}
wget -nv -O ${antlr_generator_filename} https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${antlr_generator_filename}
antlr4_tag="aacd2a2c95816d8dc1c05814051d631bfec4cf3e" # v4.6
clone https://github.com/antlr/antlr4.git antlr4 $antlr4_tag
# fix missing include
sed -i 's/^#pragma once/#pragma once\n#include <functional>/' antlr4/runtime/Cpp/runtime/src/support/CPPUtils.h
file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
antlr4_tag="5e5b6d35b4183fd330102c40947b95c4b5c6abb5" # v4.9.2
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag"
# remove shared library from install dependencies
sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/' antlr4/runtime/Cpp/runtime/CMakeLists.txt
# fix issue https://github.com/antlr/antlr4/issues/3194 - should update Antlr commit once the PR related to the issue gets merged
sed -i 's/std::is_nothrow_copy_constructible/std::is_copy_constructible/' antlr4/runtime/Cpp/runtime/src/support/Any.h
# cppitertools v2.0 2019-12-23
cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
clone https://github.com/ryanhaining/cppitertools.git cppitertools $cppitertools_ref
repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
# fmt
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
clone https://github.com/fmtlib/fmt.git fmt $fmt_tag
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag"
# rapidcheck
rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
clone https://github.com/emil-e/rapidcheck.git rapidcheck $rapidcheck_tag
repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
# google benchmark
benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
clone https://github.com/google/benchmark.git benchmark $benchmark_tag
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag"
# google test
googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
clone https://github.com/google/googletest.git googletest $googletest_tag
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag"
# google flags
gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
clone https://github.com/memgraph/gflags.git gflags $gflags_tag
repo_clone_try_double "${primary_urls[gflags]}" "${secondary_urls[gflags]}" "gflags" "$gflags_tag"
# libbcrypt
libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
clone https://github.com/rg3/libbcrypt libbcrypt $libbcrypt_tag
repo_clone_try_double "${primary_urls[libbcrypt]}" "${secondary_urls[libbcrypt]}" "libbcrypt" "$libbcrypt_tag"
# neo4j
wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz
tar -xzf neo4j.tar.gz
rm -rf neo4j
file_get_try_double "${primary_urls[neo4j]}" "${secondary_urls[neo4j]}"
tar -xzf neo4j-community-3.2.3-unix.tar.gz
mv neo4j-community-3.2.3 neo4j
rm neo4j.tar.gz
rm neo4j-community-3.2.3-unix.tar.gz
# nlohmann json
# We wget header instead of cloning repo since repo is huge (lots of test data).
# We use head on Sep 1, 2017 instead of last release since it was long time ago.
mkdir -p json
cd json
wget "https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
cd ..
bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580" # v1.0.6 (May 26, 2011)
clone https://github.com/VFR-maniac/bzip2 bzip2 $bzip2_tag
repo_clone_try_double "${primary_urls[bzip2]}" "${secondary_urls[bzip2]}" "bzip2" "$bzip2_tag"
zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
clone https://github.com/madler/zlib.git zlib $zlib_tag
repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag"
# remove shared library from install dependencies
sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
rocksdb_tag="f3e33549c151f30ac4eb7c22356c6d0331f37652" # (2020-10-14)
clone https://github.com/facebook/rocksdb.git rocksdb $rocksdb_tag
repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag"
# remove shared library from install dependencies
sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/' rocksdb/CMakeLists.txt
# mgclient
mgclient_tag="v1.2.0" # (2021-01-14)
clone https://github.com/memgraph/mgclient.git mgclient $mgclient_tag
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
# pymgclient
pymgclient_tag="4f85c179e56302d46a1e3e2cf43509db65f062b3" # (2021-01-15)
clone https://github.com/memgraph/pymgclient.git pymgclient $pymgclient_tag
repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient]}" "pymgclient" "$pymgclient_tag"
# mgconsole
mgconsole_tag="01ae99bfce772e540e75c076ba03cf06c0c2ac7d" # (2021-05-26)
repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag"
spdlog_tag="46d418164dd4cd9822cf8ca62a116a3f71569241" # (2020-12-01)
clone https://github.com/gabime/spdlog spdlog $spdlog_tag
repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag"
jemalloc_tag="ea6b3e973b477b8061e0076bb257dbd7f3faa756" # (2021-02-11)
repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$jemalloc_tag"
pushd jemalloc
# ThreadPool select job randomly, and there can be some threads that had been
# performed some memory heavy task before and will be inactive for some time,
# but until it will became active again, the memory will not be freed since by
# default each thread has it's own arena, but there should be not more then
# 4*CPU arenas (see opt.nareans description).
#
# By enabling percpu_arena number of arenas limited to number of CPUs and hence
# this problem should go away.
#
# muzzy_decay_ms -- use MADV_FREE when available on newer Linuxes, to
# avoid spurious latencies and additional work associated with
# MADV_DONTNEED. See
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
popd

5
query_modules/CMakeLists.txt
View File

@ -35,8 +35,3 @@ install(FILES graph_analyzer.py DESTINATION lib/memgraph/query_modules)
install(FILES mgp_networkx.py DESTINATION lib/memgraph/query_modules)
install(FILES nxalg.py DESTINATION lib/memgraph/query_modules)
install(FILES wcc.py DESTINATION lib/memgraph/query_modules)
if (MG_ENTERPRISE)
add_subdirectory(louvain)
add_subdirectory(connectivity)
endif()

18
query_modules/connectivity/CMakeLists.txt
View File

@ -1,18 +0,0 @@
set(MODULE src/connectivity_module.cpp)
include_directories(src)
add_library(connectivity SHARED ${MODULE})
target_include_directories(connectivity PRIVATE ${CMAKE_SOURCE_DIR}/include)
# Strip the library in release build.
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET connectivity POST_BUILD
COMMAND strip -s $<TARGET_FILE:connectivity>
COMMENT "Stripping symbols and sections from connectivity module")
endif()
install(PROGRAMS $<TARGET_FILE:connectivity>
DESTINATION lib/memgraph/query_modules
RENAME connectivity.so)

131
query_modules/connectivity/src/connectivity_module.cpp
View File

@ -1,131 +0,0 @@
#include "mg_procedure.h"
#include <queue>
#include <unordered_map>
// Finds weakly connected components of a graph.
// Time complexity: O(|V|+|E|)
static void weak(const mgp_list *args, const mgp_graph *graph,
mgp_result *result, mgp_memory *memory) {
std::unordered_map<int64_t, int64_t> vertex_component;
mgp_vertices_iterator *vertices_iterator =
mgp_graph_iter_vertices(graph, memory);
if (vertices_iterator == nullptr) {
mgp_result_set_error_msg(result, "Not enough memory");
return;
}
int64_t curr_component = 0;
for (const mgp_vertex *vertex = mgp_vertices_iterator_get(vertices_iterator);
vertex != nullptr;
vertex = mgp_vertices_iterator_next(vertices_iterator)) {
mgp_vertex_id vertex_id = mgp_vertex_get_id(vertex);
if (vertex_component.find(vertex_id.as_int) != vertex_component.end())
continue;
// run bfs from current vertex
std::queue<int64_t> q;
q.push(vertex_id.as_int);
vertex_component[vertex_id.as_int] = curr_component;
while (!q.empty()) {
mgp_vertex *v = mgp_graph_get_vertex_by_id(graph, {q.front()}, memory);
if (v == nullptr) {
mgp_vertices_iterator_destroy(vertices_iterator);
mgp_result_set_error_msg(result, "Not enough memory");
return;
}
q.pop();
// iterate over inbound edges
mgp_edges_iterator *edges_iterator = mgp_vertex_iter_in_edges(v, memory);
if (edges_iterator == nullptr) {
mgp_vertex_destroy(v);
mgp_vertices_iterator_destroy(vertices_iterator);
mgp_result_set_error_msg(result, "Not enough memory");
return;
}
for (const mgp_edge *edge = mgp_edges_iterator_get(edges_iterator);
edge != nullptr; edge = mgp_edges_iterator_next(edges_iterator)) {
mgp_vertex_id next_id = mgp_vertex_get_id(mgp_edge_get_from(edge));
if (vertex_component.find(next_id.as_int) != vertex_component.end())
continue;
vertex_component[next_id.as_int] = curr_component;
q.push(next_id.as_int);
}
// iterate over outbound edges
mgp_edges_iterator_destroy(edges_iterator);
edges_iterator = mgp_vertex_iter_out_edges(v, memory);
if (edges_iterator == nullptr) {
mgp_vertex_destroy(v);
mgp_vertices_iterator_destroy(vertices_iterator);
mgp_result_set_error_msg(result, "Not enough memory");
return;
}
for (const mgp_edge *edge = mgp_edges_iterator_get(edges_iterator);
edge != nullptr; edge = mgp_edges_iterator_next(edges_iterator)) {
mgp_vertex_id next_id = mgp_vertex_get_id(mgp_edge_get_to(edge));
if (vertex_component.find(next_id.as_int) != vertex_component.end())
continue;
vertex_component[next_id.as_int] = curr_component;
q.push(next_id.as_int);
}
mgp_vertex_destroy(v);
mgp_edges_iterator_destroy(edges_iterator);
}
++curr_component;
}
mgp_vertices_iterator_destroy(vertices_iterator);
for (const auto &p : vertex_component) {
mgp_result_record *record = mgp_result_new_record(result);
if (record == nullptr) {
mgp_result_set_error_msg(result, "Not enough memory");
return;
}
mgp_value *mem_id_value = mgp_value_make_int(p.first, memory);
if (mem_id_value == nullptr) {
mgp_result_set_error_msg(result, "Not enough memory");
return;
}
mgp_value *comp_value = mgp_value_make_int(p.second, memory);
if (comp_value == nullptr) {
mgp_value_destroy(mem_id_value);
mgp_result_set_error_msg(result, "Not enough memory");
return;
}
int mem_id_inserted = mgp_result_record_insert(record, "id", mem_id_value);
int comp_inserted =
mgp_result_record_insert(record, "component", comp_value);
mgp_value_destroy(mem_id_value);
mgp_value_destroy(comp_value);
if (!mem_id_inserted || !comp_inserted) {
mgp_result_set_error_msg(result, "Not enough memory");
return;
}
}
}
extern "C" int mgp_init_module(struct mgp_module *module,
struct mgp_memory *memory) {
struct mgp_proc *wcc_proc =
mgp_module_add_read_procedure(module, "weak", weak);
if (!mgp_proc_add_result(wcc_proc, "id", mgp_type_int())) return 1;
if (!mgp_proc_add_result(wcc_proc, "component", mgp_type_int())) return 1;
return 0;
}
extern "C" int mgp_shutdown_module() {
return 0;
}

33
query_modules/louvain/CMakeLists.txt
View File

@ -1,33 +0,0 @@
set(MAIN src/main.cpp)
set(MODULE src/louvain_module.cpp)
set(SOURCES src/algorithms/louvain.cpp
src/data_structures/graph.cpp)
include_directories(src)
add_library(louvain-core STATIC ${SOURCES})
set_target_properties(louvain-core PROPERTIES POSITION_INDEPENDENT_CODE ON)
add_executable(louvain-main ${MAIN})
target_link_libraries(louvain-main louvain-core)
enable_testing()
add_subdirectory(test)
add_library(louvain SHARED ${MODULE})
target_link_libraries(louvain louvain-core)
target_include_directories(louvain PRIVATE ${CMAKE_SOURCE_DIR}/include)
# Strip the library in release build.
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET louvain POST_BUILD
COMMAND strip -s $<TARGET_FILE:louvain>
COMMENT "Stripping symbols and sections from louvain module")
endif()
if (NOT MG_COMMUNITY)
install(PROGRAMS $<TARGET_FILE:louvain>
DESTINATION lib/memgraph/query_modules
RENAME louvain.so)
endif()

18
query_modules/louvain/src/algorithms/algorithms.hpp
View File

@ -1,18 +0,0 @@
/// @file
///
/// The file contains function declarations of several community-detection
/// graph algorithms.
#pragma once
#include "data_structures/graph.hpp"
namespace algorithms {
/// Detects communities of an unidrected, weighted graph using the Louvain
/// algorithm. The algorithm attempts to maximze the modularity of a weighted
/// graph.
///
/// @param graph pointer to an undirected, weighted graph which may contain
/// self-loops.
void Louvain(comdata::Graph *graph);
} // namespace algorithms

163
query_modules/louvain/src/algorithms/louvain.cpp
View File

@ -1,163 +0,0 @@
#include "algorithms/algorithms.hpp"
#include <algorithm>
#include <map>
#include <random>
#include <unordered_map>
namespace {
void OptimizeLocally(comdata::Graph *graph) {
// We will consider local optimizations uniformly at random.
std::random_device rd;
std::mt19937 g(rd());
std::vector<uint32_t> p(graph->Size());
std::iota(p.begin(), p.end(), 0);
std::shuffle(p.begin(), p.end(), g);
// Modularity of a graph can be expressed as:
//
// Q = 1 / (2m) * sum_over_pairs_of_nodes[(Aij - ki * kj / 2m) * delta(ci, cj)]
//
// where m is the sum of all weights in the graph,
// Aij is the weight on edge that connects i and j (i=j for a self-loop)
// ki is the sum of weights incident to node i
// ci is the community of node i
// delta(a, b) is the Kronecker delta function.
//
// With some simple algebraic manipulations, we can transform the formula into:
//
// Q = sum_over_components[M * ((sum_over_pairs(Aij + M * ki * kj)))] =
// = sum_over_components[M * (sum_over_pairs(Aij) + M * sum_over_nodes^2(ki))] =
// = sum_over_components[M * (w_contrib(ci) + M * k_contrib^2(ci))]
//
// where M = 1 / (2m)
//
// Therefore, we could store for each community the following:
// * Weight contribution (w_contrib)
// * Weighted degree contribution (k_contrib)
//
// This allows us to efficiently remove a node from one community and insert
// it into a community of its neighbour without the need to recalculate
// modularity from scratch.
std::unordered_map<uint32_t, double> w_contrib;
std::unordered_map<uint32_t, double> k_contrib;
for (uint32_t node_id = 0; node_id < graph->Size(); ++node_id) {
k_contrib[graph->Community(node_id)] += graph->IncidentWeight(node_id);
for (const auto &neigh : graph->Neighbours(node_id)) {
uint32_t nxt_id = neigh.dest;
double w = neigh.weight;
if (graph->Community(node_id) == graph->Community(nxt_id))
w_contrib[graph->Community(node_id)] += w;
}
}
bool stable = false;
double total_w = graph->TotalWeight();
while (!stable) {
stable = true;
for (uint32_t node_id : p) {
std::unordered_map<uint32_t, double> sum_w;
double self_loop = 0;
sum_w[graph->Community(node_id)] = 0;
for (const auto &neigh : graph->Neighbours(node_id)) {
uint32_t nxt_id = neigh.dest;
double weight = neigh.weight;
if (nxt_id == node_id) {
self_loop += weight;
continue;
}
sum_w[graph->Community(nxt_id)] += weight;
}
uint32_t my_c = graph->Community(node_id);
uint32_t best_c = my_c;
double best_dq = 0;
for (const auto &p : sum_w) {
if (p.first == my_c) continue;
uint32_t nxt_c = p.first;
double dq = 0;
// contributions before swap (dq = d_after - d_before)
for (uint32_t c : {my_c, nxt_c})
dq -= w_contrib[c] - k_contrib[c] * k_contrib[c] / (2.0 * total_w);
// leave the current community
dq += (w_contrib[my_c] - 2.0 * sum_w[my_c] - self_loop) -
(k_contrib[my_c] - graph->IncidentWeight(node_id)) *
(k_contrib[my_c] - graph->IncidentWeight(node_id)) /
(2.0 * total_w);
// join a new community
dq += (w_contrib[nxt_c] + 2.0 * sum_w[nxt_c] + self_loop) -
(k_contrib[nxt_c] + graph->IncidentWeight(node_id)) *
(k_contrib[nxt_c] + graph->IncidentWeight(node_id)) /
(2.0 * total_w);
if (dq > best_dq) {
best_dq = dq;
best_c = nxt_c;
}
}
if (best_c != my_c) {
graph->SetCommunity(node_id, best_c);
w_contrib[my_c] -= 2.0 * sum_w[my_c] + self_loop;
k_contrib[my_c] -= graph->IncidentWeight(node_id);
w_contrib[best_c] += 2.0 * sum_w[best_c] + self_loop;
k_contrib[best_c] += graph->IncidentWeight(node_id);
stable = false;
}
}
}
}
} // anonymous namespace
namespace algorithms {
void Louvain(comdata::Graph *graph) {
OptimizeLocally(graph);
// Collapse the locally optimized graph.
uint32_t collapsed_nodes = graph->NormalizeCommunities();
if (collapsed_nodes == graph->Size()) return;
comdata::Graph collapsed_graph(collapsed_nodes);
std::map<std::pair<uint32_t, uint32_t>, double> collapsed_edges;
for (uint32_t node_id = 0; node_id < graph->Size(); ++node_id) {
std::unordered_map<uint32_t, double> edges;
for (const auto &neigh : graph->Neighbours(node_id)) {
uint32_t nxt_id = neigh.dest;
double weight = neigh.weight;
if (graph->Community(nxt_id) < graph->Community(node_id)) continue;
edges[graph->Community(nxt_id)] += weight;
}
for (const auto &neigh : edges) {
uint32_t a = std::min(graph->Community(node_id), neigh.first);
uint32_t b = std::max(graph->Community(node_id), neigh.first);
collapsed_edges[{a, b}] += neigh.second;
}
}
for (const auto &p : collapsed_edges)
collapsed_graph.AddEdge(p.first.first, p.first.second, p.second);
// Repeat until no local optimizations can be found.
Louvain(&collapsed_graph);
// Propagate results from collapsed graph.
for (uint32_t node_id = 0; node_id < graph->Size(); ++node_id) {
graph->SetCommunity(node_id,
collapsed_graph.Community(graph->Community(node_id)));
}
graph->NormalizeCommunities();
}
} // namespace algorithms

99
query_modules/louvain/src/data_structures/graph.cpp
View File

@ -1,99 +0,0 @@
#include "data_structures/graph.hpp"
#include <exception>
#include <numeric>
#include <stdexcept>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace comdata {
Graph::Graph(uint32_t n_nodes) : n_nodes_(n_nodes), total_w_(0) {
adj_list_.resize(n_nodes, {});
inc_w_.resize(n_nodes, 0);
// each node starts as its own separate community.
community_.resize(n_nodes);
std::iota(community_.begin(), community_.end(), 0);
}
uint32_t Graph::Size() const { return n_nodes_; }
uint32_t Graph::Community(uint32_t node) const { return community_.at(node); }
void Graph::SetCommunity(uint32_t node, uint32_t c) { community_.at(node) = c; }
uint32_t Graph::NormalizeCommunities() {
std::set<uint32_t> c_id(community_.begin(), community_.end());
std::unordered_map<uint32_t, uint32_t> cmap;
uint32_t id = 0;
for (uint32_t c : c_id) {
cmap[c] = id;
++id;
}
for (uint32_t node_id = 0; node_id < n_nodes_; ++node_id)
community_[node_id] = cmap[community_[node_id]];
return id;
}
void Graph::AddEdge(uint32_t node1, uint32_t node2, double weight) {
if (node1 >= n_nodes_ || node2 >= n_nodes_)
throw std::out_of_range("Node index out of range");
if (weight <= 0) throw std::out_of_range("Weights must be positive");
if (edges_.find({node1, node2}) != edges_.end())
throw std::invalid_argument("Edge already exists");
edges_.emplace(node1, node2);
edges_.emplace(node2, node1);
total_w_ += weight;
adj_list_[node1].emplace_back(node2, weight);
inc_w_[node1] += weight;
if (node1 != node2) {
adj_list_[node2].emplace_back(node1, weight);
inc_w_[node2] += weight;
}
}
uint32_t Graph::Degree(uint32_t node) const {
return static_cast<uint32_t>(adj_list_.at(node).size());
}
double Graph::IncidentWeight(uint32_t node) const { return inc_w_.at(node); }
double Graph::TotalWeight() const { return total_w_; }
double Graph::Modularity() const {
double ret = 0;
// Since all weights should be positive, this implies that our graph has
// no edges.
if (total_w_ == 0) return 0;
std::unordered_map<uint32_t, double> weight_c;
std::unordered_map<uint32_t, double> degree_c;
for (uint32_t i = 0; i < n_nodes_; ++i) {
degree_c[Community(i)] += IncidentWeight(i);
for (const auto &neigh : adj_list_[i]) {
uint32_t j = neigh.dest;
double w = neigh.weight;
if (Community(i) != Community(j)) continue;
weight_c[Community(i)] += w;
}
}
for (const auto &p : degree_c)
ret += weight_c[p.first] - (p.second * p.second) / (2 * total_w_);
ret /= 2 * total_w_;
return ret;
}
const std::vector<Neighbour> &Graph::Neighbours(uint32_t node) const {
return adj_list_.at(node);
}
} // namespace comdata

125
query_modules/louvain/src/data_structures/graph.hpp
View File

@ -1,125 +0,0 @@
/// @file
#pragma once
#include <cstdint>
#include <set>
#include <vector>
namespace comdata {
struct Neighbour {
uint32_t dest;
double weight;
Neighbour(uint32_t d, double w) : dest(d), weight(w) {}
};
/// Class which models a weighted, undirected graph with necessary
/// functionalities for community detection algorithms.
class Graph {
public:
/// Constructs a new graph with a given number of nodes and no edges between
/// them.
///
/// The implementation assumes (and enforces) that all nodes
/// are indexed from 0 to n_nodes.
///
/// @param n_nodes Number of nodes in the graph.
explicit Graph(uint32_t n_nodes);
/// @return number of nodes in the graph.
uint32_t Size() const;
/// Adds a bidirectional, weighted edge to the graph between the given
/// nodes. If both given nodes are the same, the method inserts a weighted
/// self-loop.
///
/// There should be no edges between the given nodes when before invoking
/// this method.
///
/// @param node1 index of an incident node.
/// @param node2 index of an incident node.
/// @param weight real value which represents the weight of the edge.
///
/// @throw std::out_of_range
/// @throw std::invalid_argument
void AddEdge(uint32_t node1, uint32_t node2, double weight);
/// @param node index of node.
///
/// @return community where the node belongs to.
///
/// @throw std::out_of_range
uint32_t Community(uint32_t node) const;
/// Adds a given node to a given community.
///
/// @param node index of node.
/// @param c community where the given node should go in.
///
/// @throw std::out_of_range
void SetCommunity(uint32_t node, uint32_t c);
/// Normalizes the values of communities. More precisely, after invoking this
/// method communities will be indexed by successive integers starting from 0.
///
/// Note: this method is computationally expensive and takes O(|V|)
/// time, i.e., it traverses all nodes in the graph.
///
/// @return number of communities in the graph
uint32_t NormalizeCommunities();
/// Returns the number of incident edges to a given node. Self-loops
/// contribute a single edge to the degree.
///
/// @param node index of node.
///
/// @return degree of given node.
///
/// @throw std::out_of_range
uint32_t Degree(uint32_t node) const;
/// Returns the total weight of incident edges to a given node. Weight
/// of a self loop contributes once to the total sum.
///
/// @param node index of node.
///
/// @return total incident weight of a given node.
///
/// @throw std::out_of_range
double IncidentWeight(uint32_t node) const;
/// @return total weight of all edges in a graph.
double TotalWeight() const;
/// Calculates the modularity of the graph which is defined as a real value
/// between -1 and 1 that measures the density of links inside communities
/// compared to links between communities.
///
/// Note: this method is computationally expensive and takes O(|V| + |E|)
/// time, i.e., it traverses the entire graph.
///
/// @return modularity of the graph.
double Modularity() const;
/// Returns nodes adjacent to a given node.
///
/// @param node index of node.
///
/// @return list of neighbouring nodes.
///
/// @throw std::out_of_range
const std::vector<Neighbour>& Neighbours(uint32_t node) const;
private:
uint32_t n_nodes_;
double total_w_;
std::vector<std::vector<Neighbour>> adj_list_;
std::set<std::pair<uint32_t, uint32_t>> edges_;
std::vector<double> inc_w_;
std::vector<uint32_t> community_;
};
} // namespace comdata

228
query_modules/louvain/src/louvain_module.cpp
View File

@ -1,228 +0,0 @@
#include "mg_procedure.h"
#include <exception>
#include <string>
#include <unordered_map>
#include "algorithms/algorithms.hpp"
#include "data_structures/graph.hpp"
namespace {
std::optional<std::unordered_map<int64_t, uint32_t>> NormalizeVertexIds(
const mgp_graph *graph, mgp_result *result, mgp_memory *memory) {
std::unordered_map<int64_t, uint32_t> mem_to_louv_id;
mgp_vertices_iterator *vertices_iterator =
mgp_graph_iter_vertices(graph, memory);
if (vertices_iterator == nullptr) {
mgp_result_set_error_msg(result, "Not enough memory!");
return std::nullopt;
}
uint32_t louv_id = 0;
for (const mgp_vertex *vertex = mgp_vertices_iterator_get(vertices_iterator);
vertex != nullptr;
vertex = mgp_vertices_iterator_next(vertices_iterator)) {
mgp_vertex_id mem_id = mgp_vertex_get_id(vertex);
mem_to_louv_id[mem_id.as_int] = louv_id;
++louv_id;
}
mgp_vertices_iterator_destroy(vertices_iterator);
return mem_to_louv_id;
}
std::optional<comdata::Graph> RunLouvain(
const mgp_graph *graph, mgp_result *result, mgp_memory *memory,
const std::unordered_map<int64_t, uint32_t> &mem_to_louv_id) {
comdata::Graph louvain_graph(mem_to_louv_id.size());
// Extract the graph structure
// TODO(ipaljak): consider filtering nodes and edges by labels.
for (const auto &p : mem_to_louv_id) {
mgp_vertex *vertex = mgp_graph_get_vertex_by_id(graph, {p.first}, memory);
if (!vertex) {
mgp_result_set_error_msg(result, "Not enough memory!");
return std::nullopt;
}
// iterate over inbound edges. This is enough because we will eventually
// iterate over outbound edges in another direction.
mgp_edges_iterator *edges_iterator =
mgp_vertex_iter_in_edges(vertex, memory);
if (edges_iterator == nullptr) {
mgp_vertex_destroy(vertex);
mgp_result_set_error_msg(result, "Not enough memory!");
return std::nullopt;
}
for (const mgp_edge *edge = mgp_edges_iterator_get(edges_iterator);
edge != nullptr; edge = mgp_edges_iterator_next(edges_iterator)) {
const mgp_vertex *next_vertex = mgp_edge_get_from(edge);
mgp_vertex_id next_mem_id = mgp_vertex_get_id(next_vertex);
uint32_t next_louv_id;
try {
next_louv_id = mem_to_louv_id.at(next_mem_id.as_int);
} catch (const std::exception &e) {
const auto msg = std::string("[Internal error] ") + e.what();
mgp_result_set_error_msg(result, msg.c_str());
return std::nullopt;
}
// retrieve edge weight (default to 1)
mgp_value *weight_prop = mgp_edge_get_property(edge, "weight", memory);
if (!weight_prop) {
mgp_vertex_destroy(vertex);
mgp_edges_iterator_destroy(edges_iterator);
mgp_result_set_error_msg(result, "Not enough memory");
}
double weight = 1;
if (mgp_value_is_double(weight_prop))
weight = mgp_value_get_double(weight_prop);
if (mgp_value_is_int(weight_prop))
weight = static_cast<double>(mgp_value_get_int(weight_prop));
mgp_value_destroy(weight_prop);
try {
louvain_graph.AddEdge(p.second, next_louv_id, weight);
} catch (const std::exception &e) {
mgp_vertex_destroy(vertex);
mgp_edges_iterator_destroy(edges_iterator);
mgp_result_set_error_msg(result, e.what());
return std::nullopt;
}
}
mgp_vertex_destroy(vertex);
mgp_edges_iterator_destroy(edges_iterator);
}
try {
algorithms::Louvain(&louvain_graph);
} catch (const std::exception &e) {
const auto msg = std::string("[Internal error] ") + e.what();
mgp_result_set_error_msg(result, msg.c_str());
return std::nullopt;
}
return louvain_graph;
}
void communities(const mgp_list *args, const mgp_graph *graph,
mgp_result *result, mgp_memory *memory) {
try {
// Normalize vertex ids
auto mem_to_louv_id = NormalizeVertexIds(graph, result, memory);
if (!mem_to_louv_id) return;
// Run louvain
auto louvain_graph = RunLouvain(graph, result, memory, *mem_to_louv_id);
if (!louvain_graph) return;
// Return node ids and their corresponding communities.
for (const auto &p : *mem_to_louv_id) {
mgp_result_record *record = mgp_result_new_record(result);
if (record == nullptr) {
mgp_result_set_error_msg(result, "Not enough memory!");
return;
}
mgp_value *mem_id_value = mgp_value_make_int(p.first, memory);
if (mem_id_value == nullptr) {
mgp_result_set_error_msg(result, "Not enough memory!");
return;
}
mgp_value *com_value =
mgp_value_make_int(louvain_graph->Community(p.second), memory);
if (com_value == nullptr) {
mgp_value_destroy(mem_id_value);
mgp_result_set_error_msg(result, "Not enough memory!");
return;
}
int mem_id_inserted =
mgp_result_record_insert(record, "id", mem_id_value);
int com_inserted =
mgp_result_record_insert(record, "community", com_value);
mgp_value_destroy(mem_id_value);
mgp_value_destroy(com_value);
if (!mem_id_inserted || !com_inserted) {
mgp_result_set_error_msg(result, "Not enough memory!");
return;
}
}
} catch (const std::exception &e) {
mgp_result_set_error_msg(result, e.what());
return;
}
}
void modularity(const mgp_list *args, const mgp_graph *graph,
mgp_result *result, mgp_memory *memory) {
try {
// Normalize vertex ids
auto mem_to_louv_id = NormalizeVertexIds(graph, result, memory);
if (!mem_to_louv_id) return;
// Run louvain
auto louvain_graph = RunLouvain(graph, result, memory, *mem_to_louv_id);
if (!louvain_graph) return;
// Return graph modularity after Louvain
// TODO(ipaljak) - consider allowing the user to specify seed communities
// and
// yield modularity values both before and after running
// louvain.
mgp_result_record *record = mgp_result_new_record(result);
if (record == nullptr) {
mgp_result_set_error_msg(result, "Not enough memory!");
return;
}
mgp_value *modularity_value =
mgp_value_make_double(louvain_graph->Modularity(), memory);
if (modularity_value == nullptr) {
mgp_result_set_error_msg(result, "Not enough memory!");
return;
}
int value_inserted =
mgp_result_record_insert(record, "modularity", modularity_value);
mgp_value_destroy(modularity_value);
if (!value_inserted) {
mgp_result_set_error_msg(result, "Not enough memory!");
return;
}
} catch (const std::exception &e) {
mgp_result_set_error_msg(result, e.what());
return;
}
}
} // namespace
extern "C" int mgp_init_module(struct mgp_module *module,
struct mgp_memory *memory) {
struct mgp_proc *community_proc =
mgp_module_add_read_procedure(module, "communities", communities);
if (!community_proc) return 1;
if (!mgp_proc_add_result(community_proc, "id", mgp_type_int())) return 1;
if (!mgp_proc_add_result(community_proc, "community", mgp_type_int()))
return 1;
struct mgp_proc *modularity_proc =
mgp_module_add_read_procedure(module, "modularity", modularity);
if (!modularity_proc) return 1;
if (!mgp_proc_add_result(modularity_proc, "modularity", mgp_type_float()))
return 1;
return 0;
}
extern "C" int mgp_shutdown_module() { return 0; }

28
query_modules/louvain/src/main.cpp
View File

@ -1,28 +0,0 @@
#include <iostream>
#include "algorithms/algorithms.hpp"
#include "data_structures/graph.hpp"
// A simple program that reads the graph from STDIN and
// outputs the detected communities from louvain along with
// its modularity measure on STDOUT.
int main() {
int n;
int m;
std::cin >> n >> m;
comdata::Graph graph(n);
for (int i = 0; i < m; ++i) {
int a;
int b;
double c;
std::cin >> a >> b >> c;
graph.AddEdge(a, b, c);
}
algorithms::Louvain(&graph);
for (int i = 0; i < n; ++i)
std::cout << i << " " << graph.Community(i) << "\n";
std::cout << graph.Modularity() << "\n";
return 0;
}

80
query_modules/louvain/test/.clang-tidy
View File

@ -1,80 +0,0 @@
---
Checks: '*,
-android-*,
-cert-err58-cpp,
-cppcoreguidelines-avoid-c-arrays,
-cppcoreguidelines-avoid-goto,
-cppcoreguidelines-avoid-magic-numbers,
-cppcoreguidelines-macro-usage,
-cppcoreguidelines-no-malloc,
-cppcoreguidelines-non-private-member-variables-in-classes,
-cppcoreguidelines-owning-memory,
-cppcoreguidelines-pro-bounds-array-to-pointer-decay,
-cppcoreguidelines-pro-bounds-constant-array-index,
-cppcoreguidelines-pro-bounds-pointer-arithmetic,
-cppcoreguidelines-pro-type-member-init,
-cppcoreguidelines-pro-type-reinterpret-cast,
-cppcoreguidelines-pro-type-static-cast-downcast,
-cppcoreguidelines-pro-type-union-access,
-cppcoreguidelines-pro-type-vararg,
-cppcoreguidelines-special-member-functions,
-fuchsia-default-arguments,
-fuchsia-default-arguments-calls,
-fuchsia-default-arguments-declarations,
-fuchsia-overloaded-operator,
-fuchsia-statically-constructed-objects,
-fuchsia-trailing-return,
-fuchsia-virtual-inheritance,
-google-explicit-constructor,
-google-readability-*,
-hicpp-avoid-c-arrays,
-hicpp-avoid-goto,
-hicpp-braces-around-statements,
-hicpp-member-init,
-hicpp-no-array-decay,
-hicpp-no-assembler,
-hicpp-no-malloc,
-hicpp-special-member-functions,
-hicpp-use-equals-default,
-hicpp-vararg,
-llvm-header-guard,
-misc-non-private-member-variables-in-classes,
-misc-unused-parameters,
-modernize-avoid-c-arrays,
-modernize-concat-nested-namespaces,
-modernize-pass-by-value,
-modernize-use-equals-default,
-modernize-use-nodiscard,
-modernize-use-trailing-return-type,
-performance-unnecessary-value-param,
-readability-braces-around-statements,
-readability-else-after-return,
-readability-implicit-bool-conversion,
-readability-magic-numbers,
-readability-named-parameter'
WarningsAsErrors: ''
HeaderFilterRegex: ''
AnalyzeTemporaryDtors: false
FormatStyle: none
CheckOptions:
- key: google-readability-braces-around-statements.ShortStatementLines
value: '1'
- key: google-readability-function-size.StatementThreshold
value: '800'
- key: google-readability-namespace-comments.ShortNamespaceLines
value: '10'
- key: google-readability-namespace-comments.SpacesBeforeComments
value: '2'
- key: modernize-loop-convert.MaxCopySize
value: '16'
- key: modernize-loop-convert.MinConfidence
value: reasonable
- key: modernize-loop-convert.NamingStyle
value: CamelCase
- key: modernize-pass-by-value.IncludeStyle
value: llvm
- key: modernize-replace-auto-ptr.IncludeStyle
value: llvm
- key: modernize-use-nullptr.NullMacros
value: 'NULL'
...

3
query_modules/louvain/test/CMakeLists.txt
View File

@ -1,3 +0,0 @@
include_directories(${GTEST_INCLUDE_DIR})
add_subdirectory(unit)

28
query_modules/louvain/test/unit/CMakeLists.txt
View File

@ -1,28 +0,0 @@
set(test_prefix louvain__unit__)
add_custom_target(louvain__unit)
add_library(louvain-test STATIC utils.cpp)
set_target_properties(louvain-test PROPERTIES POSITION_INDEPENDENT_CODE ON)
function(add_unit_test test_cpp)
# get exec name (remove extension from the abs path)
get_filename_component(exec_name ${test_cpp} NAME_WE)
set(target_name ${test_prefix}${exec_name})
add_executable(${target_name} ${test_cpp})
# OUTPUT_NAME sets the real name of a target when it is built and can be
# used to help create two targets of the same name even though CMake
# requires unique logical target names
set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name})
# TODO: this is a temporary workaround the test build warnings
target_compile_options(${target_name} PRIVATE -Wno-comment -Wno-sign-compare
-Wno-unused-variable)
target_link_libraries(${target_name} spdlog gflags gtest gtest_main Threads::Threads
louvain-core louvain-test)
# register test
add_test(${target_name} ${exec_name})
# add to unit target
add_dependencies(louvain__unit ${target_name})
endfunction(add_unit_test)
add_unit_test(graph.cpp)

349
query_modules/louvain/test/unit/graph.cpp
View File

@ -1,349 +0,0 @@
#include <gtest/gtest.h>
#include "data_structures/graph.hpp"
#include "utils.hpp"
// Checks if commmunities of nodes in graph correspond to a given community
// vector.
bool CommunityCheck(const comdata::Graph &graph,
const std::vector<uint32_t> &c) {
if (graph.Size() != c.size()) return false;
for (uint32_t node_id = 0; node_id < graph.Size(); ++node_id)
if (graph.Community(node_id) != c[node_id]) return false;
return true;
}
// Checks if degrees of nodes in graph correspond to a given degree vector.
bool DegreeCheck(const comdata::Graph &graph,
const std::vector<uint32_t> &deg) {
if (graph.Size() != deg.size()) return false;
for (uint32_t node_id = 0; node_id < graph.Size(); ++node_id)
if (graph.Degree(node_id) != deg[node_id]) return false;
return true;
}
// Checks if incident weights of nodes in graph correspond to a given weight
// vector.
bool IncidentWeightCheck(const comdata::Graph &graph,
const std::vector<double> &inc_w) {
if (graph.Size() != inc_w.size()) return false;
for (uint32_t node_id = 0; node_id < graph.Size(); ++node_id)
if (std::abs(graph.IncidentWeight(node_id) - inc_w[node_id]) > 1e-6)
return false;
return true;
}
// Sets communities of nodes in graph. Returns true on success.
bool SetCommunities(comdata::Graph *graph, const std::vector<uint32_t> &c) {
if (graph->Size() != c.size()) return false;
for (uint32_t node_id = 0; node_id < graph->Size(); ++node_id)
graph->SetCommunity(node_id, c[node_id]);
return true;
}
TEST(Graph, Constructor) {
uint32_t nodes = 100;
comdata::Graph graph(nodes);
ASSERT_EQ(graph.Size(), nodes);
for (uint32_t node_id = 0; node_id < nodes; ++node_id) {
ASSERT_EQ(graph.IncidentWeight(node_id), 0);
ASSERT_EQ(graph.Community(node_id), node_id);
}
}
TEST(Graph, Size) {
comdata::Graph graph1 = GenRandomUnweightedGraph(0, 0);
comdata::Graph graph2 = GenRandomUnweightedGraph(42, 41);
comdata::Graph graph3 = GenRandomUnweightedGraph(100, 250);
ASSERT_EQ(graph1.Size(), 0);
ASSERT_EQ(graph2.Size(), 42);
ASSERT_EQ(graph3.Size(), 100);
}
TEST(Graph, Communities) {
comdata::Graph graph = GenRandomUnweightedGraph(100, 250);
for (int i = 0; i < 100; ++i) graph.SetCommunity(i, i % 5);
for (int i = 0; i < 100; ++i) ASSERT_EQ(graph.Community(i), i % 5);
// Try to set communities on non-existing nodes
EXPECT_THROW({ graph.SetCommunity(100, 2); }, std::out_of_range);
EXPECT_THROW({ graph.SetCommunity(150, 0); }, std::out_of_range);
// Try to get a the community of a non-existing node
EXPECT_THROW({ graph.Community(100); }, std::out_of_range);
EXPECT_THROW({ graph.Community(150); }, std::out_of_range);
}
TEST(Graph, CommunityNormalization) {
// Communities are already normalized.
comdata::Graph graph = GenRandomUnweightedGraph(5, 10);
std::vector<uint32_t> init_c = {0, 2, 1, 3, 4};
std::vector<uint32_t> final_c = {0, 2, 1, 3, 4};
ASSERT_TRUE(SetCommunities(&graph, init_c));
graph.NormalizeCommunities();
ASSERT_TRUE(CommunityCheck(graph, final_c));
// Each node in its own community.
graph = GenRandomUnweightedGraph(5, 10);
init_c = {20, 30, 10, 40, 50};
final_c = {1, 2, 0, 3, 4};
ASSERT_TRUE(SetCommunities(&graph, init_c));
graph.NormalizeCommunities();
ASSERT_TRUE(CommunityCheck(graph, final_c));
// Multiple nodes in the same community
graph = GenRandomUnweightedGraph(7, 10);
init_c = {13, 99, 13, 13, 1, 99, 1};
final_c = {1, 2, 1, 1, 0, 2, 0};
ASSERT_TRUE(SetCommunities(&graph, init_c));
graph.NormalizeCommunities();
ASSERT_TRUE(CommunityCheck(graph, final_c));
}
TEST(Graph, AddEdge) {
comdata::Graph graph = GenRandomUnweightedGraph(5, 0);
// Node out of bounds.
EXPECT_THROW({ graph.AddEdge(1, 5, 7); }, std::out_of_range);
// Repeated edge
graph.AddEdge(1, 2, 1);
EXPECT_THROW({ graph.AddEdge(1, 2, 7); }, std::invalid_argument);
// Non-positive edge weight
EXPECT_THROW({ graph.AddEdge(2, 3, -7); }, std::out_of_range);
EXPECT_THROW({ graph.AddEdge(3, 4, 0); }, std::out_of_range);
}
TEST(Graph, Degrees) {
// Graph without edges
comdata::Graph graph = GenRandomUnweightedGraph(5, 0);
std::vector<uint32_t> deg = {0, 0, 0, 0, 0};
ASSERT_TRUE(DegreeCheck(graph, deg));
// Chain
// (0)--(1)--(2)--(3)--(4)
graph = BuildGraph(5, {{0, 1, 1}, {1, 2, 1}, {2, 3, 1}, {3, 4, 1}});
deg = {1, 2, 2, 2, 1};
ASSERT_TRUE(DegreeCheck(graph, deg));
// Tree
// (0)--(3)
// / \
// (1) (2)
// | / \
// (4) (5) (6)
graph = BuildGraph(
7, {{0, 1, 1}, {0, 2, 1}, {0, 3, 1}, {1, 4, 1}, {2, 5, 1}, {2, 6, 1}});
deg = {3, 2, 3, 1, 1, 1, 1};
ASSERT_TRUE(DegreeCheck(graph, deg));
// Graph without self-loops
// (0)--(1)
// | \ | \
// | \ | \
// (2)--(3)-(4)
graph = BuildGraph(5, {{0, 1, 1},
{0, 2, 1},
{0, 3, 1},
{1, 3, 1},
{1, 4, 1},
{2, 3, 1},
{3, 4, 1}});
deg = {3, 3, 2, 4, 2};
ASSERT_TRUE(DegreeCheck(graph, deg));
// Graph with self loop [*nodes have self loops]
// (0)--(1*)
// | \ | \
// | \ | \
// (2*)--(3)-(4*)
graph = BuildGraph(5, {{0, 1, 1},
{0, 2, 1},
{0, 3, 1},
{1, 3, 1},
{1, 4, 1},
{2, 3, 1},
{3, 4, 1},
{1, 1, 1},
{2, 2, 2},
{4, 4, 4}});
deg = {3, 4, 3, 4, 3};
ASSERT_TRUE(DegreeCheck(graph, deg));
// Try to get degree of non-existing nodes
EXPECT_THROW({ graph.Degree(5); }, std::out_of_range);
EXPECT_THROW({ graph.Degree(100); }, std::out_of_range);
}
TEST(Graph, Weights) {
// Graph without edges
comdata::Graph graph = GenRandomUnweightedGraph(5, 0);
std::vector<double> inc_w = {0, 0, 0, 0, 0};
ASSERT_TRUE(IncidentWeightCheck(graph, inc_w));
ASSERT_EQ(graph.TotalWeight(), 0);
// Chain
// (0)--(1)--(2)--(3)--(4)
graph = BuildGraph(5, {{0, 1, 0.1}, {1, 2, 0.5}, {2, 3, 2.3}, {3, 4, 4.2}});
inc_w = {0.1, 0.6, 2.8, 6.5, 4.2};
ASSERT_TRUE(IncidentWeightCheck(graph, inc_w));
ASSERT_NEAR(graph.TotalWeight(), 7.1, 1e-6);
// Tree
// (0)--(3)
// / \
// (1) (2)
// | / \
// (4) (5) (6)
graph = BuildGraph(7, {{0, 1, 1.3},
{0, 2, 0.2},
{0, 3, 1},
{1, 4, 3.2},
{2, 5, 4.2},
{2, 6, 0.7}});
inc_w = {2.5, 4.5, 5.1, 1, 3.2, 4.2, 0.7};
ASSERT_TRUE(IncidentWeightCheck(graph, inc_w));
EXPECT_NEAR(graph.TotalWeight(), 10.6, 1e-6);
// Graph without self-loops
// (0)--(1)
// | \ | \
// | \ | \
// (2)--(3)-(4)
graph = BuildGraph(5, {{0, 1, 0.1},
{0, 2, 0.2},
{0, 3, 0.3},
{1, 3, 0.4},
{1, 4, 0.5},
{2, 3, 0.6},
{3, 4, 0.7}});
inc_w = {0.6, 1, 0.8, 2, 1.2};
ASSERT_TRUE(IncidentWeightCheck(graph, inc_w));
EXPECT_NEAR(graph.TotalWeight(), 2.8, 1e-6);
// Graph with self loop [*nodes have self loops]
// (0)--(1*)
// | \ | \
// | \ | \
// (2*)--(3)-(4*)
graph = BuildGraph(5, {{0, 1, 0.1},
{0, 2, 0.2},
{0, 3, 0.3},
{1, 3, 0.4},
{1, 4, 0.5},
{2, 3, 0.6},
{3, 4, 0.7},
{1, 1, 0.8},
{2, 2, 0.9},
{4, 4, 1}});
inc_w = {0.6, 1.8, 1.7, 2, 2.2};
ASSERT_TRUE(IncidentWeightCheck(graph, inc_w));
EXPECT_NEAR(graph.TotalWeight(), 5.5, 1e-6);
// Try to get incident weight of non-existing node
EXPECT_THROW({ graph.IncidentWeight(5); }, std::out_of_range);
EXPECT_THROW({ graph.IncidentWeight(100); }, std::out_of_range);
}
TEST(Graph, Modularity) {
// Graph without edges
comdata::Graph graph = GenRandomUnweightedGraph(5, 0);
ASSERT_EQ(graph.Modularity(), 0);
// Chain
// (0)--(1)--(2)--(3)--(4)
graph = BuildGraph(5, {{0, 1, 0.1}, {1, 2, 0.5}, {2, 3, 2.3}, {3, 4, 4.2}});
std::vector<uint32_t> c = {0, 1, 1, 2, 2};
SetCommunities(&graph, c);
EXPECT_NEAR(graph.Modularity(), 0.036798254314620096, 1e-6);
// Tree
// (0)--(3)
// / \
// (1) (2)
// | / \
// (4) (5) (6)
graph = BuildGraph(7, {{0, 1, 1.3},
{0, 2, 0.2},
{0, 3, 1},
{1, 4, 3.2},
{2, 5, 4.2},
{2, 6, 0.7}});
c = {0, 0, 1, 0, 0, 1, 2};
SetCommunities(&graph, c);
EXPECT_NEAR(graph.Modularity(), 0.4424617301530794, 1e-6);
// Graph without self-loops
// (0)--(1)
// | \ | \
// | \ | \
// (2)--(3)-(4)
graph = BuildGraph(5, {{0, 1, 0.1},
{0, 2, 0.2},
{0, 3, 0.3},
{1, 3, 0.4},
{1, 4, 0.5},
{2, 3, 0.6},
{3, 4, 0.7}});
c = {0, 1, 1, 1, 1};
SetCommunities(&graph, c);
EXPECT_NEAR(graph.Modularity(), -0.022959183673469507, 1e-6);
// Graph with self loop [*nodes have self loops]
// (0)--(1*)
// | \ | \
// | \ | \
// (2*)--(3)-(4*)
graph = BuildGraph(5, {{0, 1, 0.1},
{0, 2, 0.2},
{0, 3, 0.3},
{1, 3, 0.4},
{1, 4, 0.5},
{2, 3, 0.6},
{3, 4, 0.7},
{1, 1, 0.8},
{2, 2, 0.9},
{4, 4, 1}});
c = {0, 0, 0, 0, 1};
SetCommunities(&graph, c);
EXPECT_NEAR(graph.Modularity(), 0.188842975206611, 1e-6);
// Neo4j example graph
// (0)--(1)---(3)--(4)
// \ / \ /
// (2) (5)
graph = BuildGraph(6, {{0, 1, 1},
{1, 2, 1},
{0, 2, 1},
{1, 3, 1},
{3, 5, 1},
{5, 4, 1},
{3, 4, 1}});
c = {0, 0, 0, 1, 1, 1};
SetCommunities(&graph, c);
EXPECT_NEAR(graph.Modularity(), 0.3571428571428571, 1e-6);
// Example graph from wikipedia
// (0)--(1)--(3)--(4)--(5)
// \ / | \ /
// (2) (7) (6)
// / \
// (8)--(9)
graph = BuildGraph(10, {{0, 1, 1},
{1, 2, 1},
{0, 2, 1},
{1, 3, 1},
{3, 4, 1},
{4, 5, 1},
{5, 6, 1},
{6, 4, 1},
{3, 7, 1},
{7, 8, 1},
{7, 9, 1},
{8, 9, 1}});
c = {0, 0, 0, 0, 1, 1, 1, 2, 2, 2};
SetCommunities(&graph, c);
EXPECT_NEAR(graph.Modularity(), 0.4896, 1e-4);
}

32
query_modules/louvain/test/unit/utils.cpp
View File

@ -1,32 +0,0 @@
#include "utils.hpp"
#include <random>
comdata::Graph BuildGraph(
uint32_t nodes, std::vector<std::tuple<uint32_t, uint32_t, double>> edges) {
comdata::Graph G(nodes);
for (auto &edge : edges)
G.AddEdge(std::get<0>(edge), std::get<1>(edge), std::get<2>(edge));
return G;
}
comdata::Graph GenRandomUnweightedGraph(uint32_t nodes, uint32_t edges) {
auto seed =
std::chrono::high_resolution_clock::now().time_since_epoch().count();
std::mt19937 rng(seed);
std::uniform_int_distribution<uint32_t> dist(0, nodes - 1);
std::set<std::tuple<uint32_t, uint32_t, double>> E;
for (uint32_t i = 0; i < edges; ++i) {
int u;
int v;
do {
u = dist(rng);
v = dist(rng);
if (u > v) std::swap(u, v);
} while (u == v || E.find({u, v, 1}) != E.end());
E.insert({u, v, 1});
}
return BuildGraph(nodes, std::vector<std::tuple<uint32_t, uint32_t, double>>(
E.begin(), E.end()));
}

18
query_modules/louvain/test/unit/utils.hpp
View File

@ -1,18 +0,0 @@
#pragma once
#include <chrono>
#include <random>
#include <set>
#include <tuple>
#include "data_structures/graph.hpp"
/// Builds the graph from a given number of nodes and a list of edges.
/// Nodes should be 0-indexed and each edge should be provided only once.
comdata::Graph BuildGraph(
uint32_t nodes, std::vector<std::tuple<uint32_t, uint32_t, double>> edges);
/// Generates random undirected graph with a given number of nodes and edges.
/// The generated graph is not picked out of a uniform distribution. All weights
/// are the same and equal to one.
comdata::Graph GenRandomUnweightedGraph(uint32_t nodes, uint32_t edges);

99
release/LICENSE_COMMUNITY.md
View File

@ -1,33 +1,40 @@
# User License Agreement
# Memgraph Community User License Agreement
1. Description
This License Agreement governs your use of the Memgraph Community Release (the
"Software") and documentation ("Documentation").
THIS LICENSE AGREEMENT GOVERNS LICENSEE’S USE OF THE MEMGRAPH COMMUNITY
RELEASE AND DOCUMENTATION.
BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU ("LICENSEE") AGREE TO THESE
TERMS.
2. License Grant
1. License Grant
The Software and Documentation are provided to Licensee at no charge and are
licensed, not sold to Licensee. No ownership of any part of the Software and
Documentation is hereby transferred to Licensee. Subject to (i) the terms and
conditions of this License Agreement, (ii) any additional license restrictions
and parameters contained on Licensor’s quotation, website, or order form
(“Order Form”), Licensor hereby grants Licensee a personal, non-assignable,
conditions of this License Agreement, and (ii) any additional license
restrictions and parameters contained on Licensor’s quotation, website, or
order form, Licensor hereby grants Licensee a personal, non-assignable,
non-transferable and non-exclusive license to install, access and use the
Software (in object code form only) and Documentation for Licensee’s internal
business purposes only. All rights relating to the Software and Documentation
that are not expressly licensed in this License Agreement, whether now existing
or which may hereafter come into existence are reserved for Licensor. Licensee
shall not remove, obscure, or alter any proprietary rights notices (including
without limitation copyright and trademark notices), which may be affixed to or
contained within the Software or Documentation.
business purposes (including for use in a production environment) only. All
rights relating to the Software and Documentation that are not expressly
licensed in this License Agreement, whether now existing or which may hereafter
come into existence are reserved for Licensor. Licensee shall not remove,
obscure, or alter any proprietary rights notices (including without limitation
copyright and trademark notices), which may be affixed to or contained within
the Software or Documentation.
3. Restrictions
Licensor may terminate this License Agreement with immediate effect upon
written notice to the Licensee. Upon termination Licensee shall delete all
electronic copies of all or any part of the Software and/or the Documentation
resident in its systems or elsewhere.
2. Restrictions
Licensee will not, directly or indirectly, (a) copy the Software or
Documentation in any manner or for any purpose; (b) install, access or use any
component of the Software or Documentation for any purpose not expressly
granted in Section 2 above; (c) resell, distribute, publicly display or
granted in Section 1 above; (c) resell, distribute, publicly display or
publicly perform the Software or Documentation or any component thereof, by
transfer, lease, loan or any other means, or make it available for use by
others in any time-sharing, service bureau or similar arrangement; (d)
@ -37,25 +44,55 @@ algorithms or techniques incorporated in the Software; (e) export the Software
or Documentation in violation of any applicable laws or regulations; (f)
modify, translate, adapt, or create derivative works from the Software or
Documentation; (g) circumvent, disable or otherwise interfere with
security-related features of the Software or Documentation; (h)
reverse-engineer, disassemble, attempt to derive the source code; (i) use the
security-related features of the Software or Documentation; (h) use the
Software or Documentation for any illegal purpose, in any manner that is
inconsistent with the terms of this License Agreement, or to engage in illegal
activity; (j) remove or alter any trademark, logo, copyright or other
activity; (i) remove or alter any trademark, logo, copyright or other
proprietary notices, legends, symbols or labels on, or embedded in, the
Software or Documentation; or (k) provide access to the Software or
Software or Documentation; or (j) provide access to the Software or
Documentation to third parties.
4. Warranty Disclaimer
3. Warranty Disclaimer
THE MEMGRAPH COMMUNITY RELEASE AND DOCUMENTATION ARE PROVIDED “AS IS” FOR
DEVELOPMENT, TESTING AND EVALUATION PURPOSES ONLY. IT IS NOT LICENSED FOR
PRODUCTION USE AND LICENSOR MAKES NO AND DISCLAIMS ALL WARRANTIES, EXPRESS OR
IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NONINFRINGEMENT OF
THIRD PARTIES’ INTELLECTUAL PROPERTY RIGHTS OR OTHER PROPRIETARY RIGHTS.
NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION FURNISHED UNDER IT IS
INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE OPERATION OF THE SOFTWARE
WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" AND LICENSOR MAKES NO
WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON
INFRINGEMENT OF THIRD PARTIES’ INTELLECTUAL PROPERTY RIGHTS OR OTHER
PROPRIETARY RIGHTS. NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION
FURNISHED UNDER IT IS INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE
OPERATION OF THE SOFTWARE WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU AGREE TO SUCH TERMS.
4. Limitation of Liability
Licensor shall not in any circumstances be liable, whether in tort (including
for negligence or breach of statutory duty howsoever arising), contract,
misrepresentation (whether innocent or negligent) or otherwise for: loss of
profits, loss of business, depletion of goodwill or similar losses, loss of
anticipated savings, loss of goods, loss or corruption of data or computer
downtime, or any special, indirect, consequential or pure economic loss, costs,
damages, charges or expenses.
Licensor's total aggregate liability in contract, tort (including without
limitation negligence or breach of statutory duty howsoever arising),
misrepresentation (whether innocent or negligent), restitution or otherwise,
arising in connection with the performance or contemplated performance of this
License Agreement shall in all circumstances be limited to GBP10.00 (ten pounds
sterling).
Nothing in this License Agreement shall limit Licensor’s liability in the case
of death or personal injury caused by negligence, fraud, or fraudulent
misrepresentation, or where it otherwise cannot be limited by law.
5. Technical Data
Licensor may collect and use technical information (such as usage patterns)
gathered when the Licensee downloads and uses the Software. This is generally
statistical data which does not identify an identified or identifiable
individual. It may also include Licensee’s IP address which is personal data
and is processed in accordance with our Privacy Policy. We only use this
technical information to improve our products.
6. Law and Jurisdiction
This License Agreement is governed by the laws of England and is subject to the
non-exclusive jurisdiction of the courts of England.

2
release/arch-pkg/package_arch
View File

@ -1,7 +1,7 @@
#!/bin/bash -e
function print_help () {
echo "Usage: $0 MEMGPRAH_PACKAGE.tar.gz"
echo "Usage: $0 MEMGRAPH_PACKAGE.tar.gz"
echo "Optional arguments:"
echo -e " -h|--help Print help."
}

1
release/docker/memgraph_community.dockerfile
View File

@ -1,4 +1,5 @@
FROM debian:buster
# NOTE: If you change the base distro update release/package as well.
ARG deb_release

1
release/docker/memgraph_enterprise.dockerfile
View File

@ -1,4 +1,5 @@
FROM debian:buster
# NOTE: If you change the base distro update release/package as well.
ARG deb_release

14
release/get_version.py
View File

@ -192,7 +192,19 @@ if args.version:
try:
current_branch = get_output("git", "rev-parse", "--abbrev-ref", "HEAD")
if current_branch != "master":
get_output("git", "fetch", "origin", "master:master")
branches = get_output("git", "branch")
if "master" in branches:
# If master is present locally, the fetch is allowed to fail
# because this script will still be able to compare against the
# master branch.
try:
get_output("git", "fetch", "origin", "master:master")
except Exception:
pass
else:
# If master is not present locally, the fetch command has to
# succeed because something else will fail otherwise.
get_output("git", "fetch", "origin", "master:master")
except Exception:
print("Fatal error while ensuring local master branch.")
sys.exit(1)

12
release/package/centos-7/Dockerfile Normal file
View File

@ -0,0 +1,12 @@
FROM centos:7
RUN yum -y update \
&& yum install -y wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-centos-7.tar.gz \
-O toolchain-v2-binaries-centos-7.tar.gz \
&& tar xzvf toolchain-v2-binaries-centos-7.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

12
release/package/centos-8/Dockerfile Normal file
View File

@ -0,0 +1,12 @@
FROM centos:8
RUN dnf -y update \
&& dnf install -y wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-centos-8.tar.gz \
-O toolchain-v2-binaries-centos-8.tar.gz \
&& tar xzvf toolchain-v2-binaries-centos-8.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

15
release/package/debian-10/Dockerfile Normal file
View File

@ -0,0 +1,15 @@
FROM debian:10
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-debian-10.tar.gz \
-O toolchain-v2-binaries-debian-10.tar.gz \
&& tar xzvf toolchain-v2-binaries-debian-10.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

15
release/package/debian-9/Dockerfile Normal file
View File

@ -0,0 +1,15 @@
FROM debian:9
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-debian-9.tar.gz \
-O toolchain-v2-binaries-debian-9.tar.gz \
&& tar xzvf toolchain-v2-binaries-debian-9.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

26
release/package/docker-compose.yml Normal file
View File

@ -0,0 +1,26 @@
version: "3"
services:
mgbuild_centos-7:
build:
context: centos-7
container_name: "mgbuild_centos-7"
mgbuild_centos-8:
build:
context: centos-8
container_name: "mgbuild_centos-8"
mgbuild_debian-9:
build:
context: debian-9
container_name: "mgbuild_debian-9"
mgbuild_debian-10:
build:
context: debian-10
container_name: "mgbuild_debian-10"
mgbuild_ubuntu-18.04:
build:
context: ubuntu-18.04
container_name: "mgbuild_ubuntu-18.04"
mgbuild_ubuntu-20.04:
build:
context: ubuntu-20.04
container_name: "mgbuild_ubuntu-20.04"

153
release/package/run.sh Executable file
View File

@ -0,0 +1,153 @@
#!/bin/bash
set -Eeuo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
SUPPORTED_OFFERING=(community enterprise)
SUPPORTED_OS=(centos-7 centos-8 debian-9 debian-10 ubuntu-18.04 ubuntu-20.04)
PROJECT_ROOT="$SCRIPT_DIR/../.."
ACTIVATE_TOOLCHAIN="source /opt/toolchain-v2/activate"
HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
print_help () {
echo "$0 init|package {offering} {os} [--for-docker]|docker|test"
echo ""
echo " offerings: ${SUPPORTED_OFFERING[*]}"
echo " OSs: ${SUPPORTED_OS[*]}"
exit 1
}
make_package () {
offering="$1"
offering_flag=" -DMG_ENTERPRISE=OFF "
if [[ "$offering" == "enterprise" ]]; then
offering_flag=" -DMG_ENTERPRISE=ON "
fi
if [[ "$offering" == "community" ]]; then
offering_flag=" -DMG_ENTERPRISE=OFF "
fi
os="$2"
package_command=""
if [[ "$os" =~ ^"centos".* ]]; then
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint memgraph*.rpm "
fi
if [[ "$os" =~ ^"debian".* ]]; then
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
if [[ "$os" =~ ^"ubuntu".* ]]; then
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
docker_flag=" -DBUILD_FOR_DOCKER=OFF "
if [[ "$#" -gt 2 ]]; then
if [[ "$3" == "--for-docker" ]]; then
docker_flag=" -DBUILD_FOR_DOCKER=ON "
fi
fi
build_container="mgbuild_$os"
echo "Building Memgraph $offering for $os on $build_container..."
echo "Copying project files..."
# If master is not the current branch, fetch it, because the get_version
# script depends on it. If we are on master, the fetch command is going to
# fail so that's why there is the explicit check.
# Required here because Docker build container can't access remote.
cd "$PROJECT_ROOT"
if [[ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]]; then
git fetch origin master:master
fi
docker exec "$build_container" mkdir -p /memgraph
docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/"
container_build_dir="/memgraph/build"
container_output_dir="$container_build_dir/output"
# TODO(gitbuda): TOOLCHAIN_RUN_DEPS should be installed during the Docker
# image build phase, but that is not easy at this point because the
# environment/os/{os}.sh does not come within the toolchain package. When
# migrating to the next version of toolchain do that, and remove the
# TOOLCHAIN_RUN_DEPS installation from here.
echo "Installing dependencies..."
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
echo "Building targeted package..."
docker exec "$build_container" bash -c "cd /memgraph && ./init"
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $offering_flag $docker_flag .."
# ' is used instead of " because we need to run make within the allowed
# container resources.
# shellcheck disable=SC2016
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN "'&& make -j$(nproc)'
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN "'&& make -j$(nproc) -B mgconsole'
docker exec "$build_container" bash -c "mkdir -p $container_output_dir && cd $container_output_dir && $ACTIVATE_TOOLCHAIN && $package_command"
echo "Copying targeted package to host..."
last_package_name=$(docker exec "$build_container" bash -c "cd $container_output_dir && ls -t memgraph* | head -1")
# The operating system folder is introduced because multiple different
# packages could be preserved during the same build "session".
mkdir -p "$HOST_OUTPUT_DIR/$os"
package_host_destination="$HOST_OUTPUT_DIR/$os/$last_package_name"
docker cp "$build_container:$container_output_dir/$last_package_name" "$package_host_destination"
echo "Package saved to $package_host_destination."
}
case "$1" in
init)
cd "$SCRIPT_DIR"
docker-compose build
docker-compose up -d
;;
docker)
# NOTE: Docker is build on top of Debian 10 package.
based_on_os="debian-10"
# shellcheck disable=SC2012
last_package_name=$(cd "$HOST_OUTPUT_DIR/$based_on_os" && ls -t memgraph* | head -1)
docker_build_folder="$PROJECT_ROOT/release/docker"
cd "$docker_build_folder"
./package_deb_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
# shellcheck disable=SC2012
docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1)
docker_host_folder="$HOST_OUTPUT_DIR/docker"
docker_host_image_path="$docker_host_folder/$docker_image_name"
mkdir -p "$docker_host_folder"
cp "$docker_build_folder/$docker_image_name" "$docker_host_image_path"
echo "Docker images saved to $docker_host_image_path."
;;
package)
shift 1
if [[ "$#" -lt 2 ]]; then
print_help
fi
offering="$1"
shift 1
is_offering_ok=false
for supported_offering in "${SUPPORTED_OFFERING[@]}"; do
if [[ "$supported_offering" == "${offering}" ]]; then
is_offering_ok=true
fi
done
os="$1"
shift 1
is_os_ok=false
for supported_os in "${SUPPORTED_OS[@]}"; do
if [[ "$supported_os" == "${os}" ]]; then
is_os_ok=true
fi
done
if [[ "$is_offering_ok" == true ]] && [[ "$is_os_ok" == true ]]; then
make_package "$offering" "$os" "$@"
else
print_help
fi
;;
test)
echo "TODO(gitbuda): Test all packages on mgtest containers."
;;
*)
print_help
;;
esac

15
release/package/ubuntu-18.04/Dockerfile Normal file
View File

@ -0,0 +1,15 @@
FROM ubuntu:18.04
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-ubuntu-18.04.tar.gz \
-O toolchain-v2-binaries-ubuntu-18.04.tar.gz \
&& tar xzvf toolchain-v2-binaries-ubuntu-18.04.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

15
release/package/ubuntu-20.04/Dockerfile Normal file
View File

@ -0,0 +1,15 @@
FROM ubuntu:20.04
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-ubuntu-20.04.tar.gz \
-O toolchain-v2-binaries-ubuntu-20.04.tar.gz \
&& tar xzvf toolchain-v2-binaries-ubuntu-20.04.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

8
src/CMakeLists.txt
View File

@ -37,17 +37,17 @@ if (MG_ENTERPRISE)
glue/auth.cpp)
endif()
set(MG_SINGLE_NODE_V2_LIBS stdc++fs Threads::Threads
telemetry_lib mg-query mg-communication)
set(mg_single_node_v2_libs stdc++fs Threads::Threads
telemetry_lib mg-query mg-communication mg-new-delete mg-utils)
if (MG_ENTERPRISE)
# These are enterprise subsystems
set(MG_SINGLE_NODE_V2_LIBS ${MG_SINGLE_NODE_V2_LIBS} mg-auth mg-audit)
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-auth mg-audit)
endif()
# memgraph main executable
add_executable(memgraph ${mg_single_node_v2_sources})
target_include_directories(memgraph PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(memgraph ${MG_SINGLE_NODE_V2_LIBS})
target_link_libraries(memgraph ${mg_single_node_v2_libs})
# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which
# should be dynamically exported, so that `dlopen` can correctly link the
# symbols in custom procedure module libraries.

8
src/auth/models.cpp
View File

@ -43,6 +43,14 @@ std::string PermissionToString(Permission permission) {
return "REPLICATION";
case Permission::LOCK_PATH:
return "LOCK_PATH";
case Permission::READ_FILE:
return "READ_FILE";
case Permission::FREE_MEMORY:
return "FREE_MEMORY";
case Permission::TRIGGER:
return "TRIGGER";
case Permission::CONFIG:
return "CONFIG";
case Permission::AUTH:
return "AUTH";
}

14
src/auth/models.hpp
View File

@ -23,15 +23,21 @@ enum class Permission : uint64_t {
DUMP = 1U << 9U,
REPLICATION = 1U << 10U,
LOCK_PATH = 1U << 11U,
READ_FILE = 1U << 12U,
FREE_MEMORY = 1U << 13U,
TRIGGER = 1U << 14U,
CONFIG = 1U << 15U,
AUTH = 1U << 16U
};
// clang-format on
// Constant list of all available permissions.
const std::vector<Permission> kPermissionsAll = {
Permission::MATCH, Permission::CREATE, Permission::MERGE, Permission::DELETE, Permission::SET,
Permission::REMOVE, Permission::INDEX, Permission::STATS, Permission::CONSTRAINT, Permission::DUMP,
Permission::AUTH, Permission::REPLICATION, Permission::LOCK_PATH};
const std::vector<Permission> kPermissionsAll = {Permission::MATCH, Permission::CREATE, Permission::MERGE,
Permission::DELETE, Permission::SET, Permission::REMOVE,
Permission::INDEX, Permission::STATS, Permission::CONSTRAINT,
Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::LOCK_PATH, Permission::READ_FILE, Permission::FREE_MEMORY,
Permission::TRIGGER, Permission::CONFIG};
// Function that converts a permission to its string representation.
std::string PermissionToString(Permission permission);

2
src/auth/reference_modules/example.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/python3
import json
import io

2
src/auth/reference_modules/ldap.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/python3
import json
import io
import ssl

8
src/glue/auth.cpp
View File

@ -28,6 +28,14 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
return auth::Permission::REPLICATION;
case query::AuthQuery::Privilege::LOCK_PATH:
return auth::Permission::LOCK_PATH;
case query::AuthQuery::Privilege::READ_FILE:
return auth::Permission::READ_FILE;
case query::AuthQuery::Privilege::FREE_MEMORY:
return auth::Permission::FREE_MEMORY;
case query::AuthQuery::Privilege::TRIGGER:
return auth::Permission::TRIGGER;
case query::AuthQuery::Privilege::CONFIG:
return auth::Permission::CONFIG;
case query::AuthQuery::Privilege::AUTH:
return auth::Permission::AUTH;
}

167
src/memgraph.cpp
View File

@ -11,6 +11,7 @@
#include <optional>
#include <regex>
#include <string>
#include <string_view>
#include <thread>
#include <fmt/format.h>
@ -28,6 +29,7 @@
#include "query/procedure/module.hpp"
#include "query/procedure/py_module.hpp"
#include "requests/requests.hpp"
#include "storage/v2/isolation_level.hpp"
#include "storage/v2/storage.hpp"
#include "storage/v2/view.hpp"
#include "telemetry/telemetry.hpp"
@ -35,6 +37,8 @@
#include "utils/file.hpp"
#include "utils/flag_validation.hpp"
#include "utils/logging.hpp"
#include "utils/memory_tracker.hpp"
#include "utils/readable_size.hpp"
#include "utils/signals.hpp"
#include "utils/string.hpp"
#include "utils/sysinfo/memory.hpp"
@ -65,6 +69,42 @@
#include "glue/auth.hpp"
#endif
namespace {
std::string GetAllowedEnumValuesString(const auto &mappings) {
std::vector<std::string> allowed_values;
allowed_values.reserve(mappings.size());
std::transform(mappings.begin(), mappings.end(), std::back_inserter(allowed_values),
[](const auto &mapping) { return std::string(mapping.first); });
return utils::Join(allowed_values, ", ");
}
enum class ValidationError : uint8_t { EmptyValue, InvalidValue };
utils::BasicResult<ValidationError> IsValidEnumValueString(const auto &value, const auto &mappings) {
if (value.empty()) {
return ValidationError::EmptyValue;
}
if (std::find_if(mappings.begin(), mappings.end(), [&](const auto &mapping) { return mapping.first == value; }) ==
mappings.cend()) {
return ValidationError::InvalidValue;
}
return {};
}
template <typename Enum>
std::optional<Enum> StringToEnum(const auto &value, const auto &mappings) {
const auto mapping_iter =
std::find_if(mappings.begin(), mappings.end(), [&](const auto &mapping) { return mapping.first == value; });
if (mapping_iter == mappings.cend()) {
return std::nullopt;
}
return mapping_iter->second;
}
} // namespace
// Bolt server flags.
DEFINE_string(bolt_address, "0.0.0.0", "IP address on which the Bolt server should listen.");
DEFINE_VALIDATED_int32(bolt_port, 7687, "Port on which the Bolt server should listen.",
@ -138,6 +178,72 @@ DEFINE_uint64(query_execution_timeout_sec, 180,
"Maximum allowed query execution time. Queries exceeding this "
"limit will be aborted. Value of 0 means no limit.");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(
memory_limit, 0,
"Total memory limit in MiB. Set to 0 to use the default values which are 100\% of the phyisical memory if the swap "
"is enabled and 90\% of the physical memory otherwise.");
namespace {
using namespace std::literals;
constexpr std::array isolation_level_mappings{
std::pair{"SNAPSHOT_ISOLATION"sv, storage::IsolationLevel::SNAPSHOT_ISOLATION},
std::pair{"READ_COMMITTED"sv, storage::IsolationLevel::READ_COMMITTED},
std::pair{"READ_UNCOMMITTED"sv, storage::IsolationLevel::READ_UNCOMMITTED}};
const std::string isolation_level_help_string =
fmt::format("Default isolation level used for the transactions. Allowed values: {}",
GetAllowedEnumValuesString(isolation_level_mappings));
} // namespace
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_string(isolation_level, "SNAPSHOT_ISOLATION", isolation_level_help_string.c_str(), {
if (const auto result = IsValidEnumValueString(value, isolation_level_mappings); result.HasError()) {
const auto error = result.GetError();
switch (error) {
case ValidationError::EmptyValue: {
std::cout << "Isolation level cannot be empty." << std::endl;
break;
}
case ValidationError::InvalidValue: {
std::cout << "Invalid value for isolation level. Allowed values: "
<< GetAllowedEnumValuesString(isolation_level_mappings) << std::endl;
break;
}
}
return false;
}
return true;
});
namespace {
storage::IsolationLevel ParseIsolationLevel() {
const auto isolation_level = StringToEnum<storage::IsolationLevel>(FLAGS_isolation_level, isolation_level_mappings);
MG_ASSERT(isolation_level, "Invalid isolation level");
return *isolation_level;
}
int64_t GetMemoryLimit() {
if (FLAGS_memory_limit == 0) {
auto maybe_total_memory = utils::sysinfo::TotalMemory();
MG_ASSERT(maybe_total_memory, "Failed to fetch the total physical memory");
const auto maybe_swap_memory = utils::sysinfo::SwapTotalMemory();
MG_ASSERT(maybe_swap_memory, "Failed to fetch the total swap memory");
if (*maybe_swap_memory == 0) {
// take only 90% of the total memory
*maybe_total_memory *= 9;
*maybe_total_memory /= 10;
}
return *maybe_total_memory * 1024;
}
// We parse the memory as MiB every time
return FLAGS_memory_limit * 1024 * 1024;
}
} // namespace
namespace {
std::vector<std::filesystem::path> query_modules_directories;
} // namespace
@ -168,31 +274,28 @@ DEFINE_string(log_file, "", "Path to where the log should be stored.");
namespace {
constexpr std::array log_level_mappings{
std::pair{"TRACE", spdlog::level::trace}, std::pair{"DEBUG", spdlog::level::debug},
std::pair{"INFO", spdlog::level::info}, std::pair{"WARNING", spdlog::level::warn},
std::pair{"ERROR", spdlog::level::err}, std::pair{"CRITICAL", spdlog::level::critical}};
std::string GetAllowedLogLevelsString() {
std::vector<std::string> allowed_log_levels;
allowed_log_levels.reserve(log_level_mappings.size());
std::transform(log_level_mappings.cbegin(), log_level_mappings.cend(), std::back_inserter(allowed_log_levels),
[](const auto &mapping) { return mapping.first; });
return utils::Join(allowed_log_levels, ", ");
}
std::pair{"TRACE"sv, spdlog::level::trace}, std::pair{"DEBUG"sv, spdlog::level::debug},
std::pair{"INFO"sv, spdlog::level::info}, std::pair{"WARNING"sv, spdlog::level::warn},
std::pair{"ERROR"sv, spdlog::level::err}, std::pair{"CRITICAL"sv, spdlog::level::critical}};
const std::string log_level_help_string =
fmt::format("Minimum log level. Allowed values: {}", GetAllowedLogLevelsString());
fmt::format("Minimum log level. Allowed values: {}", GetAllowedEnumValuesString(log_level_mappings));
} // namespace
DEFINE_VALIDATED_string(log_level, "WARNING", log_level_help_string.c_str(), {
if (value.empty()) {
std::cout << "Log level cannot be empty." << std::endl;
return false;
}
if (std::find_if(log_level_mappings.cbegin(), log_level_mappings.cend(),
[&](const auto &mapping) { return mapping.first == value; }) == log_level_mappings.cend()) {
std::cout << "Invalid value for log level. Allowed values: " << GetAllowedLogLevelsString() << std::endl;
if (const auto result = IsValidEnumValueString(value, log_level_mappings); result.HasError()) {
const auto error = result.GetError();
switch (error) {
case ValidationError::EmptyValue: {
std::cout << "Log level cannot be empty." << std::endl;
break;
}
case ValidationError::InvalidValue: {
std::cout << "Invalid value for log level. Allowed values: " << GetAllowedEnumValuesString(log_level_mappings)
<< std::endl;
break;
}
}
return false;
}
@ -201,11 +304,9 @@ DEFINE_VALIDATED_string(log_level, "WARNING", log_level_help_string.c_str(), {
namespace {
void ParseLogLevel() {
const auto mapping_iter = std::find_if(log_level_mappings.cbegin(), log_level_mappings.cend(),
[](const auto &mapping) { return mapping.first == FLAGS_log_level; });
MG_ASSERT(mapping_iter != log_level_mappings.cend(), "Invalid log level");
spdlog::set_level(mapping_iter->second);
const auto log_level = StringToEnum<spdlog::level::level_enum>(FLAGS_log_level, log_level_mappings);
MG_ASSERT(log_level, "Invalid log level");
spdlog::set_level(*log_level);
}
// 5 weeks * 7 days
@ -875,10 +976,10 @@ int main(int argc, char **argv) {
// Start memory warning logger.
utils::Scheduler mem_log_scheduler;
if (FLAGS_memory_warning_threshold > 0) {
auto free_ram = utils::sysinfo::AvailableMemoryKilobytes();
auto free_ram = utils::sysinfo::AvailableMemory();
if (free_ram) {
mem_log_scheduler.Run("Memory warning", std::chrono::seconds(3), [] {
auto free_ram = utils::sysinfo::AvailableMemoryKilobytes();
auto free_ram = utils::sysinfo::AvailableMemory();
if (free_ram && *free_ram / 1024 < FLAGS_memory_warning_threshold)
spdlog::warn("Running out of available RAM, only {} MB left", *free_ram / 1024);
});
@ -924,8 +1025,11 @@ int main(int argc, char **argv) {
// End enterprise features initialization
#endif
// Main storage and execution engines initialization
const auto memory_limit = GetMemoryLimit();
spdlog::info("Memory limit set to {}", utils::GetReadableSize(memory_limit));
utils::total_memory_tracker.SetHardLimit(memory_limit);
// Main storage and execution engines initialization
storage::Config db_config{
.gc = {.type = storage::Config::Gc::Type::PERIODIC, .interval = std::chrono::seconds(FLAGS_storage_gc_cycle_sec)},
.items = {.properties_on_edges = FLAGS_storage_properties_on_edges},
@ -934,7 +1038,8 @@ int main(int argc, char **argv) {
.snapshot_retention_count = FLAGS_storage_snapshot_retention_count,
.wal_file_size_kibibytes = FLAGS_storage_wal_file_size_kib,
.wal_file_flush_every_n_tx = FLAGS_storage_wal_file_flush_every_n_tx,
.snapshot_on_exit = FLAGS_storage_snapshot_on_exit}};
.snapshot_on_exit = FLAGS_storage_snapshot_on_exit},
.transaction = {.isolation_level = ParseIsolationLevel()}};
if (FLAGS_storage_snapshot_interval_sec == 0) {
if (FLAGS_storage_wal_enabled) {
LOG_FATAL(
@ -952,7 +1057,7 @@ int main(int argc, char **argv) {
db_config.durability.snapshot_interval = std::chrono::seconds(FLAGS_storage_snapshot_interval_sec);
}
storage::Storage db(db_config);
query::InterpreterContext interpreter_context{&db};
query::InterpreterContext interpreter_context{&db, FLAGS_data_directory};
query::SetExecutionTimeout(&interpreter_context, FLAGS_query_execution_timeout_sec);
#ifdef MG_ENTERPRISE
@ -1024,5 +1129,7 @@ int main(int argc, char **argv) {
// Shutdown Python
Py_Finalize();
PyMem_RawFree(program_name);
utils::total_memory_tracker.LogPeakMemoryUsage();
return 0;
}

12
src/mg_import_csv.cpp
View File

@ -436,9 +436,9 @@ void ProcessNodeRow(storage::Storage *store, const std::vector<Field> &fields, c
} else {
pv_id = storage::PropertyValue(node_id.id);
}
auto node_property = node.SetProperty(acc.NameToProperty(field.name), pv_id);
if (!node_property.HasValue()) throw LoadException("Couldn't add property '{}' to the node", field.name);
if (!*node_property) throw LoadException("The property '{}' already exists", field.name);
auto old_node_property = node.SetProperty(acc.NameToProperty(field.name), pv_id);
if (!old_node_property.HasValue()) throw LoadException("Couldn't add property '{}' to the node", field.name);
if (!old_node_property->IsNull()) throw LoadException("The property '{}' already exists", field.name);
}
id = node_id;
} else if (field.type == "LABEL") {
@ -448,9 +448,9 @@ void ProcessNodeRow(storage::Storage *store, const std::vector<Field> &fields, c
if (!*node_label) throw LoadException("The label '{}' already exists", label);
}
} else if (field.type != "IGNORE") {
auto node_property = node.SetProperty(acc.NameToProperty(field.name), StringToValue(value, field.type));
if (!node_property.HasValue()) throw LoadException("Couldn't add property '{}' to the node", field.name);
if (!*node_property) throw LoadException("The property '{}' already exists", field.name);
auto old_node_property = node.SetProperty(acc.NameToProperty(field.name), StringToValue(value, field.type));
if (!old_node_property.HasValue()) throw LoadException("Couldn't add property '{}' to the node", field.name);
if (!old_node_property->IsNull()) throw LoadException("The property '{}' already exists", field.name);
}
}
for (const auto &label : additional_labels) {

9
src/query/CMakeLists.txt
View File

@ -9,6 +9,7 @@ add_custom_target(generate_lcp_query DEPENDS ${generated_lcp_query_files})
set(mg_query_sources
${lcp_query_cpp_files}
common.cpp
cypher_query_interpreter.cpp
dump.cpp
frontend/ast/cypher_main_visitor.cpp
frontend/ast/pretty_print.cpp
@ -17,6 +18,7 @@ set(mg_query_sources
frontend/semantic/symbol_generator.cpp
frontend/stripped.cpp
interpret/awesome_memgraph_functions.cpp
interpret/eval.cpp
interpreter.cpp
plan/operator.cpp
plan/preprocess.cpp
@ -29,13 +31,16 @@ set(mg_query_sources
procedure/mg_procedure_impl.cpp
procedure/module.cpp
procedure/py_module.cpp
serialization/property_value.cpp
trigger.cpp
trigger_context.cpp
typed_value.cpp)
add_library(mg-query STATIC ${mg_query_sources})
add_dependencies(mg-query generate_lcp_query)
target_include_directories(mg-query PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(mg-query dl cppitertools)
target_link_libraries(mg-query mg-storage-v2 mg-utils)
target_link_libraries(mg-query mg-storage-v2 mg-utils mg-kvstore)
if("${MG_PYTHON_VERSION}" STREQUAL "")
find_package(Python3 3.5 REQUIRED COMPONENTS Development)
else()
@ -67,7 +72,7 @@ add_custom_command(
OUTPUT ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include}
COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
COMMAND
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.6-complete.jar
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.9.2-complete.jar
-Dlanguage=Cpp -visitor -package antlropencypher
-o ${opencypher_generated}
${opencypher_lexer_grammar} ${opencypher_parser_grammar}

20
src/query/common.hpp
View File

@ -1,6 +1,7 @@
/// @file
#pragma once
#include <concepts>
#include <cstdint>
#include <string>
@ -10,6 +11,7 @@
#include "query/frontend/semantic/symbol.hpp"
#include "query/typed_value.hpp"
#include "storage/v2/id_types.hpp"
#include "storage/v2/property_value.hpp"
#include "storage/v2/view.hpp"
#include "utils/logging.hpp"
@ -61,15 +63,22 @@ inline void ExpectType(const Symbol &symbol, const TypedValue &value, TypedValue
throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected, symbol.name(), value.type());
}
template <typename T>
concept AccessorWithSetProperty = requires(T accessor, const storage::PropertyId key,
const storage::PropertyValue new_value) {
{ accessor.SetProperty(key, new_value) }
->std::same_as<storage::Result<storage::PropertyValue>>;
};
/// Set a property `value` mapped with given `key` on a `record`.
///
/// @throw QueryRuntimeException if value cannot be set as a property value
template <class TRecordAccessor>
void PropsSetChecked(TRecordAccessor *record, const storage::PropertyId &key, const TypedValue &value) {
template <AccessorWithSetProperty T>
storage::PropertyValue PropsSetChecked(T *record, const storage::PropertyId &key, const TypedValue &value) {
try {
auto maybe_error = record->SetProperty(key, storage::PropertyValue(value));
if (maybe_error.HasError()) {
switch (maybe_error.GetError()) {
auto maybe_old_value = record->SetProperty(key, storage::PropertyValue(value));
if (maybe_old_value.HasError()) {
switch (maybe_old_value.GetError()) {
case storage::Error::SERIALIZATION_ERROR:
throw QueryRuntimeException("Can't serialize due to concurrent operations.");
case storage::Error::DELETED_OBJECT:
@ -81,6 +90,7 @@ void PropsSetChecked(TRecordAccessor *record, const storage::PropertyId &key, co
throw QueryRuntimeException("Unexpected error when setting a property.");
}
}
return std::move(*maybe_old_value);
} catch (const TypedValueException &) {
throw QueryRuntimeException("'{}' cannot be used as a property value.", value.type());
}

21
src/query/context.hpp
View File

@ -1,10 +1,13 @@
#pragma once
#include <type_traits>
#include "query/common.hpp"
#include "query/frontend/semantic/symbol_table.hpp"
#include "query/parameters.hpp"
#include "query/plan/profile.hpp"
#include "utils/tsc.hpp"
#include "query/trigger.hpp"
#include "utils/async_timer.hpp"
namespace query {
@ -49,19 +52,25 @@ struct ExecutionContext {
DbAccessor *db_accessor{nullptr};
SymbolTable symbol_table;
EvaluationContext evaluation_context;
utils::TSCTimer execution_tsc_timer;
double max_execution_time_sec{0.0};
std::atomic<bool> *is_shutting_down{nullptr};
bool is_profile_query{false};
std::chrono::duration<double> profile_execution_time;
plan::ProfilingStats stats;
plan::ProfilingStats *stats_root{nullptr};
TriggerContextCollector *trigger_context_collector{nullptr};
utils::AsyncTimer timer;
};
static_assert(std::is_move_assignable_v<ExecutionContext>, "ExecutionContext must be move assignable!");
static_assert(std::is_move_constructible_v<ExecutionContext>, "ExecutionContext must be move constructible!");
inline bool MustAbort(const ExecutionContext &context) {
return (context.is_shutting_down && context.is_shutting_down->load(std::memory_order_acquire)) ||
(context.max_execution_time_sec > 0 &&
context.execution_tsc_timer.Elapsed() >= context.max_execution_time_sec);
return (context.is_shutting_down != nullptr && context.is_shutting_down->load(std::memory_order_acquire)) ||
context.timer.IsExpired();
}
inline plan::ProfilingStatsWithTotalTime GetStatsWithTotalTime(const ExecutionContext &context) {
return plan::ProfilingStatsWithTotalTime{context.stats, context.profile_execution_time};
}
} // namespace query

141
src/query/cypher_query_interpreter.cpp Normal file
View File

@ -0,0 +1,141 @@
#include "query/cypher_query_interpreter.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_bool(query_cost_planner, true, "Use the cost-estimating query planner.");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(query_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
FLAG_IN_RANGE(0, std::numeric_limits<int32_t>::max()));
namespace query {
CachedPlan::CachedPlan(std::unique_ptr<LogicalPlan> plan) : plan_(std::move(plan)) {}
ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::string, storage::PropertyValue> &params,
utils::SkipList<QueryCacheEntry> *cache, utils::SpinLock *antlr_lock) {
// Strip the query for caching purposes. The process of stripping a query
// "normalizes" it by replacing any literals with new parameters. This
// results in just the *structure* of the query being taken into account for
// caching.
frontend::StrippedQuery stripped_query{query_string};
// Copy over the parameters that were introduced during stripping.
Parameters parameters{stripped_query.literals()};
// Check that all user-specified parameters are provided.
for (const auto &param_pair : stripped_query.parameters()) {
auto it = params.find(param_pair.second);
if (it == params.end()) {
throw query::UnprovidedParameterError("Parameter ${} not provided.", param_pair.second);
}
parameters.Add(param_pair.first, it->second);
}
// Cache the query's AST if it isn't already.
auto hash = stripped_query.hash();
auto accessor = cache->access();
auto it = accessor.find(hash);
std::unique_ptr<frontend::opencypher::Parser> parser;
// Return a copy of both the AST storage and the query.
CachedQuery result;
bool is_cacheable = true;
auto get_information_from_cache = [&](const auto &cached_query) {
result.ast_storage.properties_ = cached_query.ast_storage.properties_;
result.ast_storage.labels_ = cached_query.ast_storage.labels_;
result.ast_storage.edge_types_ = cached_query.ast_storage.edge_types_;
result.query = cached_query.query->Clone(&result.ast_storage);
result.required_privileges = cached_query.required_privileges;
};
if (it == accessor.end()) {
{
std::unique_lock<utils::SpinLock> guard(*antlr_lock);
try {
parser = std::make_unique<frontend::opencypher::Parser>(stripped_query.query());
} catch (const SyntaxException &e) {
// There is a syntax exception in the stripped query. Re-run the parser
// on the original query to get an appropriate error messsage.
parser = std::make_unique<frontend::opencypher::Parser>(query_string);
// If an exception was not thrown here, the stripper messed something
// up.
LOG_FATAL("The stripped query can't be parsed, but the original can.");
}
}
// Convert the ANTLR4 parse tree into an AST.
AstStorage ast_storage;
frontend::ParsingContext context{true};
frontend::CypherMainVisitor visitor(context, &ast_storage);
visitor.visit(parser->tree());
if (visitor.IsCacheable()) {
CachedQuery cached_query{std::move(ast_storage), visitor.query(), query::GetRequiredPrivileges(visitor.query())};
it = accessor.insert({hash, std::move(cached_query)}).first;
get_information_from_cache(it->second);
} else {
result.ast_storage.properties_ = ast_storage.properties_;
result.ast_storage.labels_ = ast_storage.labels_;
result.ast_storage.edge_types_ = ast_storage.edge_types_;
result.query = visitor.query()->Clone(&result.ast_storage);
result.required_privileges = query::GetRequiredPrivileges(visitor.query());
is_cacheable = false;
}
} else {
get_information_from_cache(it->second);
}
return ParsedQuery{query_string,
params,
std::move(parameters),
std::move(stripped_query),
std::move(result.ast_storage),
result.query,
std::move(result.required_privileges),
is_cacheable};
}
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers) {
auto vertex_counts = plan::MakeVertexCountCache(db_accessor);
auto symbol_table = MakeSymbolTable(query, predefined_identifiers);
auto planning_context = plan::MakePlanningContext(&ast_storage, &symbol_table, query, &vertex_counts);
auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_cost_planner);
return std::make_unique<SingleNodeLogicalPlan>(std::move(root), cost, std::move(ast_storage),
std::move(symbol_table));
}
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers) {
std::optional<utils::SkipList<PlanCacheEntry>::Accessor> plan_cache_access;
if (plan_cache) {
plan_cache_access.emplace(plan_cache->access());
auto it = plan_cache_access->find(hash);
if (it != plan_cache_access->end()) {
if (it->second->IsExpired()) {
plan_cache_access->remove(hash);
} else {
return it->second;
}
}
}
auto plan = std::make_shared<CachedPlan>(
MakeLogicalPlan(std::move(ast_storage), query, parameters, db_accessor, predefined_identifiers));
if (plan_cache_access) {
plan_cache_access->insert({hash, plan});
}
return plan;
}
} // namespace query

149
src/query/cypher_query_interpreter.hpp Normal file
View File

@ -0,0 +1,149 @@
#pragma once
//////////////////////////////////////////////////////
// THIS INCLUDE SHOULD ALWAYS COME BEFORE THE
// "cypher_main_visitor.hpp"
// "planner.hpp" includes json.hpp which uses libc's
// EOF macro while "cypher_main_visitor.hpp" includes
// "antlr4-runtime.h" which contains a static variable
// of the same name, EOF.
// This hides the definition of the macro which causes
// the compilation to fail.
#include "query/plan/planner.hpp"
//////////////////////////////////////////////////////
#include "query/frontend/ast/cypher_main_visitor.hpp"
#include "query/frontend/opencypher/parser.hpp"
#include "query/frontend/semantic/required_privileges.hpp"
#include "query/frontend/semantic/symbol_generator.hpp"
#include "query/frontend/stripped.hpp"
#include "utils/flag_validation.hpp"
#include "utils/timer.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(query_cost_planner);
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_int32(query_plan_cache_ttl);
namespace query {
// TODO: Maybe this should move to query/plan/planner.
/// Interface for accessing the root operator of a logical plan.
class LogicalPlan {
public:
explicit LogicalPlan() = default;
virtual ~LogicalPlan() = default;
LogicalPlan(const LogicalPlan &) = default;
LogicalPlan &operator=(const LogicalPlan &) = default;
LogicalPlan(LogicalPlan &&) = default;
LogicalPlan &operator=(LogicalPlan &&) = default;
virtual const plan::LogicalOperator &GetRoot() const = 0;
virtual double GetCost() const = 0;
virtual const SymbolTable &GetSymbolTable() const = 0;
virtual const AstStorage &GetAstStorage() const = 0;
};
class CachedPlan {
public:
explicit CachedPlan(std::unique_ptr<LogicalPlan> plan);
const auto &plan() const { return plan_->GetRoot(); }
double cost() const { return plan_->GetCost(); }
const auto &symbol_table() const { return plan_->GetSymbolTable(); }
const auto &ast_storage() const { return plan_->GetAstStorage(); }
bool IsExpired() const {
// NOLINTNEXTLINE (modernize-use-nullptr)
return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_plan_cache_ttl);
};
private:
std::unique_ptr<LogicalPlan> plan_;
utils::Timer cache_timer_;
};
struct CachedQuery {
AstStorage ast_storage;
Query *query;
std::vector<AuthQuery::Privilege> required_privileges;
};
struct QueryCacheEntry {
bool operator==(const QueryCacheEntry &other) const { return first == other.first; }
bool operator<(const QueryCacheEntry &other) const { return first < other.first; }
bool operator==(const uint64_t &other) const { return first == other; }
bool operator<(const uint64_t &other) const { return first < other; }
uint64_t first;
// TODO: Maybe store the query string here and use it as a key with the hash
// so that we eliminate the risk of hash collisions.
CachedQuery second;
};
struct PlanCacheEntry {
bool operator==(const PlanCacheEntry &other) const { return first == other.first; }
bool operator<(const PlanCacheEntry &other) const { return first < other.first; }
bool operator==(const uint64_t &other) const { return first == other; }
bool operator<(const uint64_t &other) const { return first < other; }
uint64_t first;
// TODO: Maybe store the query string here and use it as a key with the hash
// so that we eliminate the risk of hash collisions.
std::shared_ptr<CachedPlan> second;
};
/**
* A container for data related to the parsing of a query.
*/
struct ParsedQuery {
std::string query_string;
std::map<std::string, storage::PropertyValue> user_parameters;
Parameters parameters;
frontend::StrippedQuery stripped_query;
AstStorage ast_storage;
Query *query;
std::vector<AuthQuery::Privilege> required_privileges;
bool is_cacheable{true};
};
ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::string, storage::PropertyValue> &params,
utils::SkipList<QueryCacheEntry> *cache, utils::SpinLock *antlr_lock);
class SingleNodeLogicalPlan final : public LogicalPlan {
public:
SingleNodeLogicalPlan(std::unique_ptr<plan::LogicalOperator> root, double cost, AstStorage storage,
const SymbolTable &symbol_table)
: root_(std::move(root)), cost_(cost), storage_(std::move(storage)), symbol_table_(symbol_table) {}
const plan::LogicalOperator &GetRoot() const override { return *root_; }
double GetCost() const override { return cost_; }
const SymbolTable &GetSymbolTable() const override { return symbol_table_; }
const AstStorage &GetAstStorage() const override { return storage_; }
private:
std::unique_ptr<plan::LogicalOperator> root_;
double cost_;
AstStorage storage_;
SymbolTable symbol_table_;
};
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers);
/**
* Return the parsed *Cypher* query's AST cached logical plan, or create and
* cache a fresh one if it doesn't yet exist.
* @param predefined_identifiers optional identifiers you want to inject into a query.
* If an identifier is not defined in a scope, we check the predefined identifiers.
* If an identifier is contained there, we inject it at that place and remove it,
* because a predefined identifier can be used only in one scope.
*/
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers = {});
} // namespace query

84
src/query/db_accessor.hpp
View File

@ -43,6 +43,8 @@ class EdgeAccessor final {
public:
explicit EdgeAccessor(storage::EdgeAccessor impl) : impl_(std::move(impl)) {}
bool IsVisible(storage::View view) const { return impl_.IsVisible(view); }
storage::EdgeTypeId EdgeType() const { return impl_.EdgeType(); }
auto Properties(storage::View view) const { return impl_.Properties(view); }
@ -51,16 +53,16 @@ class EdgeAccessor final {
return impl_.GetProperty(key, view);
}
storage::Result<bool> SetProperty(storage::PropertyId key, const storage::PropertyValue &value) {
storage::Result<storage::PropertyValue> SetProperty(storage::PropertyId key, const storage::PropertyValue &value) {
return impl_.SetProperty(key, value);
}
storage::Result<bool> RemoveProperty(storage::PropertyId key) { return SetProperty(key, storage::PropertyValue()); }
storage::Result<storage::PropertyValue> RemoveProperty(storage::PropertyId key) {
return SetProperty(key, storage::PropertyValue());
}
utils::BasicResult<storage::Error, void> ClearProperties() {
auto ret = impl_.ClearProperties();
if (ret.HasError()) return ret.GetError();
return {};
storage::Result<std::map<storage::PropertyId, storage::PropertyValue>> ClearProperties() {
return impl_.ClearProperties();
}
VertexAccessor To() const;
@ -87,6 +89,8 @@ class VertexAccessor final {
public:
explicit VertexAccessor(storage::VertexAccessor impl) : impl_(std::move(impl)) {}
bool IsVisible(storage::View view) const { return impl_.IsVisible(view); }
auto Labels(storage::View view) const { return impl_.Labels(view); }
storage::Result<bool> AddLabel(storage::LabelId label) { return impl_.AddLabel(label); }
@ -103,16 +107,16 @@ class VertexAccessor final {
return impl_.GetProperty(key, view);
}
storage::Result<bool> SetProperty(storage::PropertyId key, const storage::PropertyValue &value) {
storage::Result<storage::PropertyValue> SetProperty(storage::PropertyId key, const storage::PropertyValue &value) {
return impl_.SetProperty(key, value);
}
storage::Result<bool> RemoveProperty(storage::PropertyId key) { return SetProperty(key, storage::PropertyValue()); }
storage::Result<storage::PropertyValue> RemoveProperty(storage::PropertyId key) {
return SetProperty(key, storage::PropertyValue());
}
utils::BasicResult<storage::Error, void> ClearProperties() {
auto ret = impl_.ClearProperties();
if (ret.HasError()) return ret.GetError();
return {};
storage::Result<std::map<storage::PropertyId, storage::PropertyValue>> ClearProperties() {
return impl_.ClearProperties();
}
auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
@ -208,6 +212,8 @@ class DbAccessor final {
return std::nullopt;
}
void FinalizeTransaction() { accessor_->FinalizeTransaction(); }
VerticesIterable Vertices(storage::View view) { return VerticesIterable(accessor_->Vertices(view)); }
VerticesIterable Vertices(storage::View view, storage::LabelId label) {
@ -235,17 +241,59 @@ class DbAccessor final {
const storage::EdgeTypeId &edge_type) {
auto maybe_edge = accessor_->CreateEdge(&from->impl_, &to->impl_, edge_type);
if (maybe_edge.HasError()) return storage::Result<EdgeAccessor>(maybe_edge.GetError());
return EdgeAccessor(std::move(*maybe_edge));
return EdgeAccessor(*maybe_edge);
}
storage::Result<bool> RemoveEdge(EdgeAccessor *edge) { return accessor_->DeleteEdge(&edge->impl_); }
storage::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
auto res = accessor_->DeleteEdge(&edge->impl_);
if (res.HasError()) {
return res.GetError();
}
storage::Result<bool> DetachRemoveVertex(VertexAccessor *vertex_accessor) {
return accessor_->DetachDeleteVertex(&vertex_accessor->impl_);
const auto &value = res.GetValue();
if (!value) {
return std::optional<EdgeAccessor>{};
}
return std::make_optional<EdgeAccessor>(*value);
}
storage::Result<bool> RemoveVertex(VertexAccessor *vertex_accessor) {
return accessor_->DeleteVertex(&vertex_accessor->impl_);
storage::Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
VertexAccessor *vertex_accessor) {
using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
auto res = accessor_->DetachDeleteVertex(&vertex_accessor->impl_);
if (res.HasError()) {
return res.GetError();
}
const auto &value = res.GetValue();
if (!value) {
return std::optional<ReturnType>{};
}
const auto &[vertex, edges] = *value;
std::vector<EdgeAccessor> deleted_edges;
deleted_edges.reserve(edges.size());
std::transform(edges.begin(), edges.end(), std::back_inserter(deleted_edges),
[](const auto &deleted_edge) { return EdgeAccessor{deleted_edge}; });
return std::make_optional<ReturnType>(vertex, std::move(deleted_edges));
}
storage::Result<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
auto res = accessor_->DeleteVertex(&vertex_accessor->impl_);
if (res.HasError()) {
return res.GetError();
}
const auto &value = res.GetValue();
if (!value) {
return std::optional<VertexAccessor>{};
}
return std::make_optional<VertexAccessor>(*value);
}
storage::PropertyId NameToProperty(const std::string_view &name) { return accessor_->NameToProperty(name); }

19
src/query/exceptions.hpp
View File

@ -161,7 +161,24 @@ class ReplicationModificationInMulticommandTxException : public QueryException {
class LockPathModificationInMulticommandTxException : public QueryException {
public:
LockPathModificationInMulticommandTxException()
: QueryException("Lock path clause not allowed in multicommand transactions.") {}
: QueryException("Lock path query not allowed in multicommand transactions.") {}
};
class FreeMemoryModificationInMulticommandTxException : public QueryException {
public:
FreeMemoryModificationInMulticommandTxException()
: QueryException("Free memory query not allowed in multicommand transactions.") {}
};
class TriggerModificationInMulticommandTxException : public QueryException {
public:
TriggerModificationInMulticommandTxException()
: QueryException("Trigger queries not allowed in multicommand transactions.") {}
};
class IsolationLevelModificationInMulticommandTxException : public QueryException {
public:
IsolationLevelModificationInMulticommandTxException()
: QueryException("Isolation level cannot be modified in multicommand transactions.") {}
};
} // namespace query

115
src/query/frontend/ast/ast.lcp
View File

@ -686,9 +686,7 @@ cpp<#
symbol_pos_ = symbol.position();
return this;
}
cpp<#)
(:protected
#>cpp
explicit Identifier(const std::string &name) : name_(name) {}
Identifier(const std::string &name, bool user_declared)
: name_(name), user_declared_(user_declared) {}
@ -1543,7 +1541,11 @@ cpp<#
:scope :public
:slk-save #'slk-save-ast-vector
:slk-load (slk-load-ast-vector "CypherUnion")
:documentation "Contains remaining queries that should form and union with `single_query_`."))
:documentation "Contains remaining queries that should form and union with `single_query_`.")
(memory-limit "Expression *" :initval "nullptr" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression"))
(memory-scale "size_t" :initval "1024U" :scope :public))
(:public
#>cpp
CypherQuery() = default;
@ -2191,7 +2193,7 @@ cpp<#
(:serialize))
(lcp:define-enum privilege
(create delete match merge set remove index stats auth constraint
dump replication lock_path)
dump replication lock_path read_file free_memory trigger config)
(:serialize))
#>cpp
AuthQuery() = default;
@ -2228,7 +2230,10 @@ const std::vector<AuthQuery::Privilege> kPrivilegesAll = {
AuthQuery::Privilege::AUTH,
AuthQuery::Privilege::CONSTRAINT, AuthQuery::Privilege::DUMP,
AuthQuery::Privilege::REPLICATION,
AuthQuery::Privilege::LOCK_PATH};
AuthQuery::Privilege::READ_FILE,
AuthQuery::Privilege::LOCK_PATH,
AuthQuery::Privilege::FREE_MEMORY, AuthQuery::Privilege::TRIGGER,
AuthQuery::Privilege::CONFIG};
cpp<#
(lcp:define-class info-query (query)
@ -2353,4 +2358,102 @@ cpp<#
(:serialize (:slk))
(:clone))
(lcp:define-class load-csv (clause)
((file "Expression *" :scope :public)
(with_header "bool" :scope :public)
(ignore_bad "bool" :scope :public)
(delimiter "Expression *" :initval "nullptr" :scope :public)
(quote "Expression *" :initval "nullptr" :scope :public)
(row_var "Identifier *" :initval "nullptr" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Identifier")))
(:public
#>cpp
LoadCsv() = default;
bool Accept(HierarchicalTreeVisitor &visitor) override {
if (visitor.PreVisit(*this)) {
row_var_->Accept(visitor);
}
return visitor.PostVisit(*this);
}
cpp<#)
(:protected
#>cpp
explicit LoadCsv(Expression *file, bool with_header, bool ignore_bad, Expression *delimiter,
Expression* quote, Identifier* row_var)
: file_(file),
with_header_(with_header),
ignore_bad_(ignore_bad),
delimiter_(delimiter),
quote_(quote),
row_var_(row_var) {
DMG_ASSERT(row_var, "LoadCsv cannot take nullptr for identifier");
}
cpp<#)
(:private
#>cpp
friend class AstStorage;
cpp<#)
(:serialize (:slk))
(:clone))
(lcp:define-class free-memory-query (query) ()
(:public
#>cpp
DEFVISITABLE(QueryVisitor<void>);
cpp<#)
(:serialize (:slk))
(:clone))
(lcp:define-class trigger-query (query)
((action "Action" :scope :public)
(event_type "EventType" :scope :public)
(trigger_name "std::string" :scope :public)
(before_commit "bool" :scope :public)
(statement "std::string" :scope :public))
(:public
(lcp:define-enum action
(create-trigger drop-trigger show-triggers)
(:serialize))
(lcp:define-enum event-type
(any vertex_create edge_create create vertex_delete edge_delete delete vertex_update edge_update update)
(:serialize))
#>cpp
TriggerQuery() = default;
DEFVISITABLE(QueryVisitor<void>);
cpp<#)
(:private
#>cpp
friend class AstStorage;
cpp<#)
(:serialize (:slk))
(:clone))
(lcp:define-class isolation-level-query (query)
((isolation_level "IsolationLevel" :scope :public)
(isolation_level_scope "IsolationLevelScope" :scope :public))
(:public
(lcp:define-enum isolation-level
(snapshot-isolation read-committed read-uncommitted)
(:serialize))
(lcp:define-enum isolation-level-scope
(next session global)
(:serialize))
#>cpp
IsolationLevelQuery() = default;
DEFVISITABLE(QueryVisitor<void>);
cpp<#)
(:private
#>cpp
friend class AstStorage;
cpp<#)
(:serialize (:slk))
(:clone))
(lcp:pop-namespace) ;; namespace query

9
src/query/frontend/ast/ast_visitor.hpp
View File

@ -74,6 +74,10 @@ class RegexMatch;
class DumpQuery;
class ReplicationQuery;
class LockPathQuery;
class LoadCsv;
class FreeMemoryQuery;
class TriggerQuery;
class IsolationLevelQuery;
using TreeCompositeVisitor = ::utils::CompositeVisitor<
SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
@ -82,7 +86,7 @@ using TreeCompositeVisitor = ::utils::CompositeVisitor<
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral, MapLiteral,
PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any, None, CallProcedure,
Create, Match, Return, With, Pattern, NodeAtom, EdgeAtom, Delete, Where, SetProperty, SetProperties, SetLabels,
RemoveProperty, RemoveLabels, Merge, Unwind, RegexMatch>;
RemoveProperty, RemoveLabels, Merge, Unwind, RegexMatch, LoadCsv>;
using TreeLeafVisitor = ::utils::LeafVisitor<Identifier, PrimitiveLiteral, ParameterLookup>;
@ -106,6 +110,7 @@ class ExpressionVisitor
template <class TResult>
class QueryVisitor : public ::utils::Visitor<TResult, CypherQuery, ExplainQuery, ProfileQuery, IndexQuery, AuthQuery,
InfoQuery, ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery> {};
InfoQuery, ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery,
FreeMemoryQuery, TriggerQuery, IsolationLevelQuery> {};
} // namespace query

219
src/query/frontend/ast/cypher_main_visitor.cpp
View File

@ -33,6 +33,28 @@ namespace query::frontend {
const std::string CypherMainVisitor::kAnonPrefix = "anon";
namespace {
template <typename TVisitor>
std::optional<std::pair<query::Expression *, size_t>> VisitMemoryLimit(
MemgraphCypher::MemoryLimitContext *memory_limit_ctx, TVisitor *visitor) {
MG_ASSERT(memory_limit_ctx);
if (memory_limit_ctx->UNLIMITED()) {
return std::nullopt;
}
auto memory_limit = memory_limit_ctx->literal()->accept(visitor);
size_t memory_scale = 1024U;
if (memory_limit_ctx->MB()) {
memory_scale = 1024U * 1024U;
} else {
MG_ASSERT(memory_limit_ctx->KB());
memory_scale = 1024U;
}
return std::make_pair(memory_limit, memory_scale);
}
} // namespace
antlrcpp::Any CypherMainVisitor::visitExplainQuery(MemgraphCypher::ExplainQueryContext *ctx) {
MG_ASSERT(ctx->children.size() == 2, "ExplainQuery should have exactly two children!");
auto *cypher_query = ctx->children[1]->accept(this).as<CypherQuery *>();
@ -127,6 +149,14 @@ antlrcpp::Any CypherMainVisitor::visitCypherQuery(MemgraphCypher::CypherQueryCon
cypher_query->cypher_unions_.push_back(child->accept(this).as<CypherUnion *>());
}
if (auto *memory_limit_ctx = ctx->queryMemoryLimit()) {
const auto memory_limit_info = VisitMemoryLimit(memory_limit_ctx->memoryLimit(), this);
if (memory_limit_info) {
cypher_query->memory_limit_ = memory_limit_info->first;
cypher_query->memory_scale_ = memory_limit_info->second;
}
}
query_ = cypher_query;
return cypher_query;
}
@ -263,6 +293,152 @@ antlrcpp::Any CypherMainVisitor::visitLockPathQuery(MemgraphCypher::LockPathQuer
return lock_query;
}
antlrcpp::Any CypherMainVisitor::visitLoadCsv(MemgraphCypher::LoadCsvContext *ctx) {
auto *load_csv = storage_->Create<LoadCsv>();
// handle file name
if (ctx->csvFile()->literal()->StringLiteral()) {
load_csv->file_ = ctx->csvFile()->accept(this);
} else {
throw SemanticException("CSV file path should be a string literal");
}
// handle header options
// Don't have to check for ctx->HEADER(), as it's a mandatory token.
// Just need to check if ctx->WITH() is not nullptr - otherwise, we have a
// ctx->NO() and ctx->HEADER() present.
load_csv->with_header_ = ctx->WITH() != nullptr;
// handle skip bad row option
load_csv->ignore_bad_ = ctx->IGNORE() && ctx->BAD();
// handle delimiter
if (ctx->DELIMITER()) {
if (ctx->delimiter()->literal()->StringLiteral()) {
load_csv->delimiter_ = ctx->delimiter()->accept(this);
} else {
throw SemanticException("Delimiter should be a string literal");
}
}
// handle quote
if (ctx->QUOTE()) {
if (ctx->quote()->literal()->StringLiteral()) {
load_csv->quote_ = ctx->quote()->accept(this);
} else {
throw SemanticException("Quote should be a string literal");
}
}
// handle row variable
load_csv->row_var_ = storage_->Create<Identifier>(ctx->rowVar()->variable()->accept(this).as<std::string>());
return load_csv;
}
antlrcpp::Any CypherMainVisitor::visitFreeMemoryQuery(MemgraphCypher::FreeMemoryQueryContext *ctx) {
auto *free_memory_query = storage_->Create<FreeMemoryQuery>();
query_ = free_memory_query;
return free_memory_query;
}
antlrcpp::Any CypherMainVisitor::visitTriggerQuery(MemgraphCypher::TriggerQueryContext *ctx) {
MG_ASSERT(ctx->children.size() == 1, "TriggerQuery should have exactly one child!");
auto *trigger_query = ctx->children[0]->accept(this).as<TriggerQuery *>();
query_ = trigger_query;
return trigger_query;
}
antlrcpp::Any CypherMainVisitor::visitCreateTrigger(MemgraphCypher::CreateTriggerContext *ctx) {
auto *trigger_query = storage_->Create<TriggerQuery>();
trigger_query->action_ = TriggerQuery::Action::CREATE_TRIGGER;
trigger_query->trigger_name_ = ctx->triggerName()->symbolicName()->accept(this).as<std::string>();
auto *statement = ctx->triggerStatement();
antlr4::misc::Interval interval{statement->start->getStartIndex(), statement->stop->getStopIndex()};
trigger_query->statement_ = ctx->start->getInputStream()->getText(interval);
trigger_query->event_type_ = [ctx] {
if (!ctx->ON()) {
return TriggerQuery::EventType::ANY;
}
if (ctx->CREATE(1)) {
if (ctx->emptyVertex()) {
return TriggerQuery::EventType::VERTEX_CREATE;
}
if (ctx->emptyEdge()) {
return TriggerQuery::EventType::EDGE_CREATE;
}
return TriggerQuery::EventType::CREATE;
}
if (ctx->DELETE()) {
if (ctx->emptyVertex()) {
return TriggerQuery::EventType::VERTEX_DELETE;
}
if (ctx->emptyEdge()) {
return TriggerQuery::EventType::EDGE_DELETE;
}
return TriggerQuery::EventType::DELETE;
}
if (ctx->UPDATE()) {
if (ctx->emptyVertex()) {
return TriggerQuery::EventType::VERTEX_UPDATE;
}
if (ctx->emptyEdge()) {
return TriggerQuery::EventType::EDGE_UPDATE;
}
return TriggerQuery::EventType::UPDATE;
}
LOG_FATAL("Invalid token allowed for the query");
}();
trigger_query->before_commit_ = ctx->BEFORE();
return trigger_query;
}
antlrcpp::Any CypherMainVisitor::visitDropTrigger(MemgraphCypher::DropTriggerContext *ctx) {
auto *trigger_query = storage_->Create<TriggerQuery>();
trigger_query->action_ = TriggerQuery::Action::DROP_TRIGGER;
trigger_query->trigger_name_ = ctx->triggerName()->symbolicName()->accept(this).as<std::string>();
return trigger_query;
}
antlrcpp::Any CypherMainVisitor::visitShowTriggers(MemgraphCypher::ShowTriggersContext *ctx) {
auto *trigger_query = storage_->Create<TriggerQuery>();
trigger_query->action_ = TriggerQuery::Action::SHOW_TRIGGERS;
return trigger_query;
}
antlrcpp::Any CypherMainVisitor::visitIsolationLevelQuery(MemgraphCypher::IsolationLevelQueryContext *ctx) {
auto *isolation_level_query = storage_->Create<IsolationLevelQuery>();
isolation_level_query->isolation_level_scope_ = [scope = ctx->isolationLevelScope()]() {
if (scope->GLOBAL()) {
return IsolationLevelQuery::IsolationLevelScope::GLOBAL;
}
if (scope->SESSION()) {
return IsolationLevelQuery::IsolationLevelScope::SESSION;
}
return IsolationLevelQuery::IsolationLevelScope::NEXT;
}();
isolation_level_query->isolation_level_ = [level = ctx->isolationLevel()]() {
if (level->SNAPSHOT()) {
return IsolationLevelQuery::IsolationLevel::SNAPSHOT_ISOLATION;
}
if (level->COMMITTED()) {
return IsolationLevelQuery::IsolationLevel::READ_COMMITTED;
}
return IsolationLevelQuery::IsolationLevel::READ_UNCOMMITTED;
}();
query_ = isolation_level_query;
return isolation_level_query;
}
antlrcpp::Any CypherMainVisitor::visitCypherUnion(MemgraphCypher::CypherUnionContext *ctx) {
bool distinct = !ctx->ALL();
auto *cypher_union = storage_->Create<CypherUnion>(distinct);
@ -292,6 +468,7 @@ antlrcpp::Any CypherMainVisitor::visitSingleQuery(MemgraphCypher::SingleQueryCon
bool has_return = false;
bool has_optional_match = false;
bool has_call_procedure = false;
bool has_load_csv = false;
for (Clause *clause : single_query->clauses_) {
const auto &clause_type = clause->GetTypeInfo();
@ -304,6 +481,14 @@ antlrcpp::Any CypherMainVisitor::visitSingleQuery(MemgraphCypher::SingleQueryCon
if (has_update || has_return) {
throw SemanticException("UNWIND can't be put after RETURN clause or after an update.");
}
} else if (utils::IsSubtype(clause_type, LoadCsv::kType)) {
if (has_load_csv) {
throw SemanticException("Can't have multiple LOAD CSV clauses in a single query.");
}
if (has_return) {
throw SemanticException("LOAD CSV can't be put after RETURN clause.");
}
has_load_csv = true;
} else if (auto *match = utils::Downcast<Match>(clause)) {
if (has_update || has_return) {
throw SemanticException("MATCH can't be put after RETURN clause or after an update.");
@ -388,6 +573,9 @@ antlrcpp::Any CypherMainVisitor::visitClause(MemgraphCypher::ClauseContext *ctx)
if (ctx->callProcedure()) {
return static_cast<Clause *>(ctx->callProcedure()->accept(this).as<CallProcedure *>());
}
if (ctx->loadCsv()) {
return static_cast<Clause *>(ctx->loadCsv()->accept(this).as<LoadCsv *>());
}
// TODO: implement other clauses.
throw utils::NotYetImplemented("clause '{}'", ctx->getText());
return 0;
@ -410,6 +598,14 @@ antlrcpp::Any CypherMainVisitor::visitCreate(MemgraphCypher::CreateContext *ctx)
}
antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedureContext *ctx) {
// Don't cache queries which call procedures because the
// procedure definition can affect the behaviour of the visitor and
// the execution of the query.
// If a user recompiles and reloads the procedure with different result
// names, because of the cache, old result names will be expected while the
// procedure will return results mapped to new names.
is_cacheable_ = false;
auto *call_proc = storage_->Create<CallProcedure>();
MG_ASSERT(!ctx->procedureName()->symbolicName().empty());
std::vector<std::string> procedure_subnames;
@ -422,21 +618,19 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur
for (auto *expr : ctx->expression()) {
call_proc->arguments_.push_back(expr->accept(this));
}
if (auto *memory_limit_ctx = ctx->callProcedureMemoryLimit()) {
if (memory_limit_ctx->LIMIT()) {
call_proc->memory_limit_ = memory_limit_ctx->literal()->accept(this);
if (memory_limit_ctx->MB()) {
call_proc->memory_scale_ = 1024U * 1024U;
} else {
MG_ASSERT(memory_limit_ctx->KB());
call_proc->memory_scale_ = 1024U;
}
if (auto *memory_limit_ctx = ctx->procedureMemoryLimit()) {
const auto memory_limit_info = VisitMemoryLimit(memory_limit_ctx->memoryLimit(), this);
if (memory_limit_info) {
call_proc->memory_limit_ = memory_limit_info->first;
call_proc->memory_scale_ = memory_limit_info->second;
}
} else {
// Default to 100 MB
call_proc->memory_limit_ = storage_->Create<PrimitiveLiteral>(TypedValue(100));
call_proc->memory_scale_ = 1024U * 1024U;
}
auto *yield_ctx = ctx->yieldProcedureResults();
if (!yield_ctx) {
const auto &maybe_found =
@ -493,6 +687,7 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur
// fields removed, then the query execution will report an error that we are
// yielding missing fields. The user can then just retry the query.
}
return call_proc;
}
@ -671,6 +866,12 @@ antlrcpp::Any CypherMainVisitor::visitPrivilege(MemgraphCypher::PrivilegeContext
if (ctx->AUTH()) return AuthQuery::Privilege::AUTH;
if (ctx->CONSTRAINT()) return AuthQuery::Privilege::CONSTRAINT;
if (ctx->DUMP()) return AuthQuery::Privilege::DUMP;
if (ctx->REPLICATION()) return AuthQuery::Privilege::REPLICATION;
if (ctx->LOCK_PATH()) return AuthQuery::Privilege::LOCK_PATH;
if (ctx->READ_FILE()) return AuthQuery::Privilege::READ_FILE;
if (ctx->FREE_MEMORY()) return AuthQuery::Privilege::FREE_MEMORY;
if (ctx->TRIGGER()) return AuthQuery::Privilege::TRIGGER;
if (ctx->CONFIG()) return AuthQuery::Privilege::CONFIG;
LOG_FATAL("Should not get here - unknown privilege!");
}

39
src/query/frontend/ast/cypher_main_visitor.hpp
View File

@ -208,6 +208,41 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
*/
antlrcpp::Any visitLockPathQuery(MemgraphCypher::LockPathQueryContext *ctx) override;
/**
* @return LoadCsvQuery*
*/
antlrcpp::Any visitLoadCsv(MemgraphCypher::LoadCsvContext *ctx) override;
/**
* @return FreeMemoryQuery*
*/
antlrcpp::Any visitFreeMemoryQuery(MemgraphCypher::FreeMemoryQueryContext *ctx) override;
/**
* @return TriggerQuery*
*/
antlrcpp::Any visitTriggerQuery(MemgraphCypher::TriggerQueryContext *ctx) override;
/**
* @return CreateTrigger*
*/
antlrcpp::Any visitCreateTrigger(MemgraphCypher::CreateTriggerContext *ctx) override;
/**
* @return DropTrigger*
*/
antlrcpp::Any visitDropTrigger(MemgraphCypher::DropTriggerContext *ctx) override;
/**
* @return ShowTriggers*
*/
antlrcpp::Any visitShowTriggers(MemgraphCypher::ShowTriggersContext *ctx) override;
/**
* @return IsolationLevelQuery*
*/
antlrcpp::Any visitIsolationLevelQuery(MemgraphCypher::IsolationLevelQueryContext *ctx) override;
/**
* @return CypherUnion*
*/
@ -693,6 +728,8 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
Query *query() { return query_; }
const static std::string kAnonPrefix;
bool IsCacheable() const { return is_cacheable_; }
private:
LabelIx AddLabel(const std::string &name);
PropertyIx AddProperty(const std::string &name);
@ -710,6 +747,8 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
// We use this variable in visitReturnItem to check if we are in with or
// return.
bool in_with_ = false;
bool is_cacheable_ = true;
};
} // namespace frontend
} // namespace query

14
src/query/frontend/opencypher/grammar/Cypher.g4
View File

@ -52,7 +52,7 @@ explainQuery : EXPLAIN cypherQuery ;
profileQuery : PROFILE cypherQuery ;
cypherQuery : singleQuery ( cypherUnion )* ;
cypherQuery : singleQuery ( cypherUnion )* ( queryMemoryLimit )? ;
indexQuery : createIndex | dropIndex;
@ -106,14 +106,18 @@ with : WITH ( DISTINCT )? returnBody ( where )? ;
cypherReturn : RETURN ( DISTINCT )? returnBody ;
callProcedure : CALL procedureName '(' ( expression ( ',' expression )* )? ')' ( callProcedureMemoryLimit )? ( yieldProcedureResults )? ;
callProcedure : CALL procedureName '(' ( expression ( ',' expression )* )? ')' ( procedureMemoryLimit )? ( yieldProcedureResults )? ;
procedureName : symbolicName ( '.' symbolicName )* ;
callProcedureMemoryLimit : MEMORY ( UNLIMITED | LIMIT literal ( MB | KB ) ) ;
yieldProcedureResults : YIELD ( '*' | ( procedureResult ( ',' procedureResult )* ) ) ;
memoryLimit : MEMORY ( UNLIMITED | LIMIT literal ( MB | KB ) ) ;
queryMemoryLimit : QUERY memoryLimit ;
procedureMemoryLimit : PROCEDURE memoryLimit ;
procedureResult : ( variable AS variable ) | variable ;
returnBody : returnItems ( order )? ( skip )? ( limit )? ;
@ -357,7 +361,9 @@ cypherKeyword : ALL
| OPTIONAL
| OR
| ORDER
| PROCEDURE
| PROFILE
| QUERY
| REDUCE
| REMOVE
| RETURN

2
src/query/frontend/opencypher/grammar/CypherLexer.g4
View File

@ -118,7 +118,9 @@ ON : O N ;
OPTIONAL : O P T I O N A L ;
OR : O R ;
ORDER : O R D E R ;
PROCEDURE : P R O C E D U R E ;
PROFILE : P R O F I L E ;
QUERY : Q U E R Y ;
REDUCE : R E D U C E ;
REMOVE : R E M O V E ;
RETURN : R E T U R N ;

104
src/query/frontend/opencypher/grammar/MemgraphCypher.g4
View File

@ -7,24 +7,43 @@ options { tokenVocab=MemgraphCypherLexer; }
import Cypher ;
memgraphCypherKeyword : cypherKeyword
| AFTER
| ALTER
| ASYNC
| AUTH
| BAD
| BEFORE
| CLEAR
| CONFIG
| CSV
| COMMIT
| COMMITTED
| DATA
| DELIMITER
| DATABASE
| DENY
| DROP
| DUMP
| EXECUTE
| FOR
| FREE
| FROM
| GLOBAL
| GRANT
| HEADER
| IDENTIFIED
| ISOLATION
| LEVEL
| LOAD
| LOCK
| MAIN
| MODE
| NEXT
| NO
| PASSWORD
| PORT
| PRIVILEGES
| READ
| REGISTER
| REPLICA
| REPLICAS
@ -32,11 +51,19 @@ memgraphCypherKeyword : cypherKeyword
| REVOKE
| ROLE
| ROLES
| QUOTE
| SESSION
| SNAPSHOT
| STATS
| SYNC
| TRANSACTION
| TRIGGER
| TRIGGERS
| TIMEOUT
| TO
| UNCOMMITTED
| UNLOCK
| UPDATE
| USER
| USERS
;
@ -56,6 +83,9 @@ query : cypherQuery
| dumpQuery
| replicationQuery
| lockPathQuery
| freeMemoryQuery
| triggerQuery
| isolationLevelQuery
;
authQuery : createRole
@ -82,6 +112,38 @@ replicationQuery : setReplicationRole
| showReplicas
;
triggerQuery : createTrigger
| dropTrigger
| showTriggers
;
clause : cypherMatch
| unwind
| merge
| create
| set
| cypherDelete
| remove
| with
| cypherReturn
| callProcedure
| loadCsv
;
loadCsv : LOAD CSV FROM csvFile ( WITH | NO ) HEADER
( IGNORE BAD ) ?
( DELIMITER delimiter ) ?
( QUOTE quote ) ?
AS rowVar ;
csvFile : literal ;
delimiter : literal ;
quote : literal ;
rowVar : variable ;
userOrRoleName : symbolicName ;
createRole : CREATE ROLE role=userOrRoleName ;
@ -109,8 +171,24 @@ denyPrivilege : DENY ( ALL PRIVILEGES | privileges=privilegeList ) TO userOrRole
revokePrivilege : REVOKE ( ALL PRIVILEGES | privileges=privilegeList ) FROM userOrRole=userOrRoleName ;
privilege : CREATE | DELETE | MATCH | MERGE | SET
| REMOVE | INDEX | STATS | AUTH | CONSTRAINT | DUMP ;
privilege : CREATE
| DELETE
| MATCH
| MERGE
| SET
| REMOVE
| INDEX
| STATS
| AUTH
| CONSTRAINT
| DUMP
| REPLICATION
| LOCK_PATH
| READ_FILE
| FREE_MEMORY
| TRIGGER
| CONFIG
;
privilegeList : privilege ( ',' privilege )* ;
@ -141,3 +219,25 @@ showReplicas : SHOW REPLICAS ;
lockPathQuery : ( LOCK | UNLOCK ) DATA DIRECTORY ;
freeMemoryQuery : FREE MEMORY ;
triggerName : symbolicName ;
triggerStatement : .*? ;
emptyVertex : '(' ')' ;
emptyEdge : dash dash rightArrowHead ;
createTrigger : CREATE TRIGGER triggerName ( ON ( emptyVertex | emptyEdge ) ? ( CREATE | UPDATE | DELETE ) ) ?
( AFTER | BEFORE ) COMMIT EXECUTE triggerStatement ;
dropTrigger : DROP TRIGGER triggerName ;
showTriggers : SHOW TRIGGERS ;
isolationLevel : SNAPSHOT ISOLATION | READ COMMITTED | READ UNCOMMITTED ;
isolationLevelScope : GLOBAL | SESSION | NEXT ;
isolationLevelQuery : SET isolationLevelScope TRANSACTION ISOLATION LEVEL isolationLevel ;

32
src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4
View File

@ -10,27 +10,51 @@ lexer grammar MemgraphCypherLexer ;
import CypherLexer ;
UNDERSCORE : '_' ;
AFTER : A F T E R ;
ALTER : A L T E R ;
ASYNC : A S Y N C ;
AUTH : A U T H ;
BAD : B A D ;
BEFORE : B E F O R E ;
CLEAR : C L E A R ;
COMMIT : C O M M I T ;
COMMITTED : C O M M I T T E D ;
CONFIG : C O N F I G ;
CSV : C S V ;
DATA : D A T A ;
DELIMITER : D E L I M I T E R ;
DATABASE : D A T A B A S E ;
DENY : D E N Y ;
DIRECTORY : D I R E C T O R Y ;
DROP : D R O P ;
DUMP : D U M P ;
EXECUTE : E X E C U T E ;
FOR : F O R ;
FREE : F R E E ;
FREE_MEMORY : F R E E UNDERSCORE M E M O R Y ;
FROM : F R O M ;
GLOBAL : G L O B A L ;
GRANT : G R A N T ;
GRANTS : G R A N T S ;
HEADER : H E A D E R ;
IDENTIFIED : I D E N T I F I E D ;
IGNORE : I G N O R E ;
ISOLATION : I S O L A T I O N ;
LEVEL : L E V E L ;
LOAD : L O A D ;
LOCK : L O C K ;
LOCK_PATH : L O C K UNDERSCORE P A T H ;
MAIN : M A I N ;
MODE : M O D E ;
NEXT : N E X T ;
NO : N O ;
PASSWORD : P A S S W O R D ;
PORT : P O R T ;
PRIVILEGES : P R I V I L E G E S ;
READ : R E A D ;
READ_FILE : R E A D UNDERSCORE F I L E ;
REGISTER : R E G I S T E R ;
REPLICA : R E P L I C A ;
REPLICAS : R E P L I C A S ;
@ -38,10 +62,18 @@ REPLICATION : R E P L I C A T I O N ;
REVOKE : R E V O K E ;
ROLE : R O L E ;
ROLES : R O L E S ;
QUOTE : Q U O T E ;
SESSION : S E S S I O N ;
SNAPSHOT : S N A P S H O T ;
STATS : S T A T S ;
SYNC : S Y N C ;
TIMEOUT : T I M E O U T ;
TO : T O ;
TRANSACTION : T R A N S A C T I O N ;
TRIGGER : T R I G G E R ;
TRIGGERS : T R I G G E R S ;
UNCOMMITTED : U N C O M M I T T E D ;
UNLOCK : U N L O C K ;
UPDATE : U P D A T E ;
USER : U S E R ;
USERS : U S E R S ;

4
src/query/frontend/opencypher/parser.hpp
View File

@ -35,7 +35,7 @@ class Parser {
private:
class FirstMessageErrorListener : public antlr4::BaseErrorListener {
void syntaxError(antlr4::IRecognizer *, antlr4::Token *, size_t line, size_t position, const std::string &message,
void syntaxError(antlr4::Recognizer *, antlr4::Token *, size_t line, size_t position, const std::string &message,
std::exception_ptr) override {
if (error_.empty()) {
error_ = "line " + std::to_string(line) + ":" + std::to_string(position + 1) + " " + message;
@ -48,7 +48,7 @@ class Parser {
FirstMessageErrorListener error_listener_;
std::string query_;
antlr4::ANTLRInputStream input_{query_.c_str()};
antlr4::ANTLRInputStream input_{query_};
antlropencypher::MemgraphCypherLexer lexer_{&input_};
antlr4::CommonTokenStream tokens_{&lexer_};

57
src/query/frontend/semantic/required_privileges.cpp
View File

@ -1,4 +1,5 @@
#include "query/frontend/ast/ast.hpp"
#include "query/frontend/ast/ast_visitor.hpp"
namespace query {
@ -50,70 +51,62 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
void Visit(LockPathQuery &lock_path_query) override { AddPrivilege(AuthQuery::Privilege::LOCK_PATH); }
void Visit(ReplicationQuery &replication_query) override {
switch (replication_query.action_) {
case ReplicationQuery::Action::SET_REPLICATION_ROLE:
AddPrivilege(AuthQuery::Privilege::REPLICATION);
break;
case ReplicationQuery::Action::SHOW_REPLICATION_ROLE:
AddPrivilege(AuthQuery::Privilege::REPLICATION);
break;
case ReplicationQuery::Action::REGISTER_REPLICA:
AddPrivilege(AuthQuery::Privilege::REPLICATION);
break;
case ReplicationQuery::Action::DROP_REPLICA:
AddPrivilege(AuthQuery::Privilege::REPLICATION);
break;
case ReplicationQuery::Action::SHOW_REPLICAS:
AddPrivilege(AuthQuery::Privilege::REPLICATION);
break;
}
}
void Visit(FreeMemoryQuery &free_memory_query) override { AddPrivilege(AuthQuery::Privilege::FREE_MEMORY); }
bool PreVisit(Create &) override {
void Visit(TriggerQuery &trigger_query) override { AddPrivilege(AuthQuery::Privilege::TRIGGER); }
void Visit(ReplicationQuery &replication_query) override { AddPrivilege(AuthQuery::Privilege::REPLICATION); }
void Visit(IsolationLevelQuery &isolation_level_query) override { AddPrivilege(AuthQuery::Privilege::CONFIG); }
bool PreVisit(Create & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::CREATE);
return false;
}
bool PreVisit(CallProcedure &) override {
bool PreVisit(CallProcedure & /*unused*/) override {
// TODO: Corresponding privilege
return false;
}
bool PreVisit(Delete &) override {
bool PreVisit(Delete & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::DELETE);
return false;
}
bool PreVisit(Match &) override {
bool PreVisit(Match & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::MATCH);
return false;
}
bool PreVisit(Merge &) override {
bool PreVisit(Merge & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::MERGE);
return false;
}
bool PreVisit(SetProperty &) override {
bool PreVisit(SetProperty & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::SET);
return false;
}
bool PreVisit(SetProperties &) override {
bool PreVisit(SetProperties & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::SET);
return false;
}
bool PreVisit(SetLabels &) override {
bool PreVisit(SetLabels & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::SET);
return false;
}
bool PreVisit(RemoveProperty &) override {
bool PreVisit(RemoveProperty & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::REMOVE);
return false;
}
bool PreVisit(RemoveLabels &) override {
bool PreVisit(RemoveLabels & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::REMOVE);
return false;
}
bool PreVisit(LoadCsv & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::READ_FILE);
return false;
}
bool Visit(Identifier &) override { return true; }
bool Visit(PrimitiveLiteral &) override { return true; }
bool Visit(ParameterLookup &) override { return true; }
bool Visit(Identifier & /*unused*/) override { return true; }
bool Visit(PrimitiveLiteral & /*unused*/) override { return true; }
bool Visit(ParameterLookup & /*unused*/) override { return true; }
private:
void AddPrivilege(AuthQuery::Privilege privilege) {

52
src/query/frontend/semantic/symbol_generator.cpp
View File

@ -12,8 +12,23 @@
namespace query {
namespace {
std::unordered_map<std::string, Identifier *> GeneratePredefinedIdentifierMap(
const std::vector<Identifier *> &predefined_identifiers) {
std::unordered_map<std::string, Identifier *> identifier_map;
for (const auto &identifier : predefined_identifiers) {
identifier_map.emplace(identifier->name_, identifier);
}
return identifier_map;
}
} // namespace
SymbolGenerator::SymbolGenerator(SymbolTable *symbol_table, const std::vector<Identifier *> &predefined_identifiers)
: symbol_table_(symbol_table), predefined_identifiers_{GeneratePredefinedIdentifierMap(predefined_identifiers)} {}
auto SymbolGenerator::CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type, int token_position) {
auto symbol = symbol_table_.CreateSymbol(name, user_declared, type, token_position);
auto symbol = symbol_table_->CreateSymbol(name, user_declared, type, token_position);
scope_.symbols[name] = symbol;
return symbol;
}
@ -162,6 +177,16 @@ bool SymbolGenerator::PostVisit(CallProcedure &call_proc) {
return true;
}
bool SymbolGenerator::PreVisit(LoadCsv &load_csv) { return false; }
bool SymbolGenerator::PostVisit(LoadCsv &load_csv) {
if (HasSymbol(load_csv.row_var_->name_)) {
throw RedeclareVariableError(load_csv.row_var_->name_);
}
load_csv.row_var_->MapTo(CreateSymbol(load_csv.row_var_->name_, true));
return true;
}
bool SymbolGenerator::PreVisit(Return &ret) {
scope_.in_return = true;
VisitReturnBody(ret.body_);
@ -217,7 +242,8 @@ bool SymbolGenerator::PostVisit(Match &) {
// Check variables in property maps after visiting Match, so that they can
// reference symbols out of bind order.
for (auto &ident : scope_.identifiers_in_match) {
if (!HasSymbol(ident->name_)) throw UnboundVariableError(ident->name_);
if (!HasSymbol(ident->name_) && !ConsumePredefinedIdentifier(ident->name_))
throw UnboundVariableError(ident->name_);
ident->MapTo(scope_.symbols[ident->name_]);
}
scope_.identifiers_in_match.clear();
@ -267,7 +293,7 @@ SymbolGenerator::ReturnType SymbolGenerator::Visit(Identifier &ident) {
scope_.identifiers_in_match.emplace_back(&ident);
} else {
// Everything else references a bound symbol.
if (!HasSymbol(ident.name_)) throw UnboundVariableError(ident.name_);
if (!HasSymbol(ident.name_) && !ConsumePredefinedIdentifier(ident.name_)) throw UnboundVariableError(ident.name_);
symbol = scope_.symbols[ident.name_];
}
ident.MapTo(symbol);
@ -438,10 +464,10 @@ bool SymbolGenerator::PreVisit(EdgeAtom &edge_atom) {
// Create inner symbols, but don't bind them in scope, since they are to
// be used in the missing filter expression.
auto *inner_edge = edge_atom.filter_lambda_.inner_edge;
inner_edge->MapTo(symbol_table_.CreateSymbol(inner_edge->name_, inner_edge->user_declared_, Symbol::Type::EDGE));
inner_edge->MapTo(symbol_table_->CreateSymbol(inner_edge->name_, inner_edge->user_declared_, Symbol::Type::EDGE));
auto *inner_node = edge_atom.filter_lambda_.inner_node;
inner_node->MapTo(
symbol_table_.CreateSymbol(inner_node->name_, inner_node->user_declared_, Symbol::Type::VERTEX));
symbol_table_->CreateSymbol(inner_node->name_, inner_node->user_declared_, Symbol::Type::VERTEX));
}
if (edge_atom.weight_lambda_.expression) {
VisitWithIdentifiers(edge_atom.weight_lambda_.expression,
@ -496,4 +522,20 @@ void SymbolGenerator::VisitWithIdentifiers(Expression *expr, const std::vector<I
bool SymbolGenerator::HasSymbol(const std::string &name) { return scope_.symbols.find(name) != scope_.symbols.end(); }
bool SymbolGenerator::ConsumePredefinedIdentifier(const std::string &name) {
auto it = predefined_identifiers_.find(name);
if (it == predefined_identifiers_.end()) {
return false;
}
// we can only use the predefined identifier in a single scope so we remove it after creating
// a symbol for it
auto &identifier = it->second;
MG_ASSERT(!identifier->user_declared_, "Predefined symbols cannot be user declared!");
identifier->MapTo(CreateSymbol(identifier->name_, identifier->user_declared_));
predefined_identifiers_.erase(it);
return true;
}
} // namespace query

17
src/query/frontend/semantic/symbol_generator.hpp
View File

@ -17,7 +17,7 @@ namespace query {
/// variable types.
class SymbolGenerator : public HierarchicalTreeVisitor {
public:
explicit SymbolGenerator(SymbolTable &symbol_table) : symbol_table_(symbol_table) {}
explicit SymbolGenerator(SymbolTable *symbol_table, const std::vector<Identifier *> &predefined_identifiers);
using HierarchicalTreeVisitor::PostVisit;
using HierarchicalTreeVisitor::PreVisit;
@ -36,6 +36,8 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
bool PostVisit(Create &) override;
bool PreVisit(CallProcedure &) override;
bool PostVisit(CallProcedure &) override;
bool PreVisit(LoadCsv &) override;
bool PostVisit(LoadCsv &) override;
bool PreVisit(Return &) override;
bool PostVisit(Return &) override;
bool PreVisit(With &) override;
@ -114,6 +116,9 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
bool HasSymbol(const std::string &name);
// @return true if it added a predefined identifier with that name
bool ConsumePredefinedIdentifier(const std::string &name);
// Returns a freshly generated symbol. Previous mapping of the same name to a
// different symbol is replaced with the new one.
auto CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY,
@ -127,15 +132,19 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
void VisitWithIdentifiers(Expression *, const std::vector<Identifier *> &);
SymbolTable &symbol_table_;
SymbolTable *symbol_table_;
// Identifiers which are injected from outside the query. Each identifier
// is mapped by its name.
std::unordered_map<std::string, Identifier *> predefined_identifiers_;
Scope scope_;
std::unordered_set<std::string> prev_return_names_;
std::unordered_set<std::string> curr_return_names_;
};
inline SymbolTable MakeSymbolTable(CypherQuery *query) {
inline SymbolTable MakeSymbolTable(CypherQuery *query, const std::vector<Identifier *> &predefined_identifiers = {}) {
SymbolTable symbol_table;
SymbolGenerator symbol_generator(symbol_table);
SymbolGenerator symbol_generator(&symbol_table, predefined_identifiers);
query->single_query_->Accept(symbol_generator);
for (auto *cypher_union : query->cypher_unions_) {
cypher_union->Accept(symbol_generator);

17
src/query/frontend/stripped.cpp
View File

@ -35,6 +35,7 @@ StrippedQuery::StrippedQuery(const std::string &query) : original_(query) {
};
std::vector<std::pair<Token, std::string>> tokens;
std::string unstripped_chunk;
for (int i = 0; i < static_cast<int>(original_.size());) {
Token token = Token::UNMATCHED;
int len = 0;
@ -58,6 +59,13 @@ StrippedQuery::StrippedQuery(const std::string &query) : original_(query) {
if (token == Token::UNMATCHED) throw LexingException("Invalid query.");
tokens.emplace_back(token, original_.substr(i, len));
i += len;
// if we notice execute, we create a trigger which has defined statements
// the statements will be parsed separately later on so we skip it for now
if (utils::IEquals(tokens.back().second, "execute")) {
unstripped_chunk = original_.substr(i);
break;
}
}
std::vector<std::string> token_strings;
@ -79,6 +87,7 @@ StrippedQuery::StrippedQuery(const std::string &query) : original_(query) {
// named expressions in return.
for (int i = 0; i < static_cast<int>(tokens.size()); ++i) {
auto &token = tokens[i];
// We need to shift token index for every parameter since antlr's parser
// thinks of parameter as two tokens.
int token_index = token_strings.size() + parameters_.size();
@ -123,6 +132,10 @@ StrippedQuery::StrippedQuery(const std::string &query) : original_(query) {
}
}
if (!unstripped_chunk.empty()) {
token_strings.push_back(std::move(unstripped_chunk));
}
query_ = utils::Join(token_strings, " ");
hash_ = utils::Fnv(query_);
@ -156,6 +169,7 @@ StrippedQuery::StrippedQuery(const std::string &query) : original_(query) {
}
// There is only whitespace, nothing to do...
if (it == tokens.end()) break;
bool has_as = false;
auto last_non_space = it;
auto jt = it;
@ -168,7 +182,8 @@ StrippedQuery::StrippedQuery(const std::string &query) : original_(query) {
for (;
jt != tokens.end() && (jt->second != "," || num_open_braces || num_open_parantheses || num_open_brackets) &&
!utils::IEquals(jt->second, "order") && !utils::IEquals(jt->second, "skip") &&
!utils::IEquals(jt->second, "limit") && !utils::IEquals(jt->second, "union") && jt->second != ";";
!utils::IEquals(jt->second, "limit") && !utils::IEquals(jt->second, "union") &&
!utils::IEquals(jt->second, "query") && jt->second != ";";
++jt) {
if (jt->second == "(") {
++num_open_parantheses;

22
src/query/frontend/stripped_lexer_constants.hpp
View File

@ -79,15 +79,19 @@ class Trie {
const int kBitsetSize = 65536;
const trie::Trie kKeywords = {
"union", "all", "optional", "match", "unwind", "as", "merge", "on", "create",
"set", "detach", "delete", "remove", "with", "distinct", "return", "order", "by",
"skip", "limit", "ascending", "asc", "descending", "desc", "where", "or", "xor",
"and", "not", "in", "starts", "ends", "contains", "is", "null", "case",
"when", "then", "else", "end", "count", "filter", "extract", "any", "none",
"single", "true", "false", "reduce", "coalesce", "user", "password", "alter", "drop",
"show", "stats", "unique", "explain", "profile", "storage", "index", "info", "exists",
"assert", "constraint", "node", "key", "dump", "database", "call", "yield", "memory",
"mb", "kb", "unlimited"};
"union", "all", "optional", "match", "unwind", "as", "merge", "on",
"create", "set", "detach", "delete", "remove", "with", "distinct", "return",
"order", "by", "skip", "limit", "ascending", "asc", "descending", "desc",
"where", "or", "xor", "and", "not", "in", "starts", "ends",
"contains", "is", "null", "case", "when", "then", "else", "end",
"count", "filter", "extract", "any", "none", "single", "true", "false",
"reduce", "coalesce", "user", "password", "alter", "drop", "show", "stats",
"unique", "explain", "profile", "storage", "index", "info", "exists", "assert",
"constraint", "node", "key", "dump", "database", "call", "yield", "memory",
"mb", "kb", "unlimited", "free", "procedure", "query", "free_memory", "read_file",
"lock_path", "after", "before", "execute", "transaction", "trigger", "triggers", "update",
"comitted", "uncomitted", "global", "isolation", "level", "next", "read", "session",
"snapshot", "transaction"};
// Unicode codepoints that are allowed at the start of the unescaped name.
const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(

24
src/query/interpret/eval.cpp Normal file
View File

@ -0,0 +1,24 @@
#include "query/interpret/eval.hpp"
namespace query {
int64_t EvaluateInt(ExpressionEvaluator *evaluator, Expression *expr, const std::string &what) {
TypedValue value = expr->Accept(*evaluator);
try {
return value.ValueInt();
} catch (TypedValueException &e) {
throw QueryRuntimeException(what + " must be an int");
}
}
std::optional<size_t> EvaluateMemoryLimit(ExpressionEvaluator *eval, Expression *memory_limit, size_t memory_scale) {
if (!memory_limit) return std::nullopt;
auto limit_value = memory_limit->Accept(*eval);
if (!limit_value.IsInt() || limit_value.ValueInt() <= 0)
throw QueryRuntimeException("Memory limit must be a non-negative integer.");
size_t limit = limit_value.ValueInt();
if (std::numeric_limits<size_t>::max() / memory_scale < limit) throw QueryRuntimeException("Memory limit overflow.");
return limit * memory_scale;
}
} // namespace query

11
src/query/interpret/eval.hpp
View File

@ -656,13 +656,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
/// @param what - Name of what's getting evaluated. Used for user feedback (via
/// exception) when the evaluated value is not an int.
/// @throw QueryRuntimeException if expression doesn't evaluate to an int.
inline int64_t EvaluateInt(ExpressionEvaluator *evaluator, Expression *expr, const std::string &what) {
TypedValue value = expr->Accept(*evaluator);
try {
return value.ValueInt();
} catch (TypedValueException &e) {
throw QueryRuntimeException(what + " must be an int");
}
}
int64_t EvaluateInt(ExpressionEvaluator *evaluator, Expression *expr, const std::string &what);
std::optional<size_t> EvaluateMemoryLimit(ExpressionEvaluator *eval, Expression *memory_limit, size_t memory_scale);
} // namespace query

776
src/query/interpreter.cpp
View File

File diff suppressed because it is too large Load Diff

97
src/query/interpreter.hpp
View File

@ -3,6 +3,7 @@
#include <gflags/gflags.h>
#include "query/context.hpp"
#include "query/cypher_query_interpreter.hpp"
#include "query/db_accessor.hpp"
#include "query/exceptions.hpp"
#include "query/frontend/ast/ast.hpp"
@ -12,18 +13,18 @@
#include "query/plan/operator.hpp"
#include "query/plan/read_write_type_checker.hpp"
#include "query/stream.hpp"
#include "query/trigger.hpp"
#include "query/typed_value.hpp"
#include "storage/v2/isolation_level.hpp"
#include "utils/event_counter.hpp"
#include "utils/logging.hpp"
#include "utils/memory.hpp"
#include "utils/skip_list.hpp"
#include "utils/spin_lock.hpp"
#include "utils/thread_pool.hpp"
#include "utils/timer.hpp"
#include "utils/tsc.hpp"
DECLARE_bool(query_cost_planner);
DECLARE_int32(query_plan_cache_ttl);
namespace EventCounter {
extern const Event FailedQuery;
} // namespace EventCounter
@ -99,11 +100,11 @@ class ReplicationQueryHandler {
ReplicationQueryHandler() = default;
virtual ~ReplicationQueryHandler() = default;
ReplicationQueryHandler(const ReplicationQueryHandler &) = delete;
ReplicationQueryHandler &operator=(const ReplicationQueryHandler &) = delete;
ReplicationQueryHandler(const ReplicationQueryHandler &) = default;
ReplicationQueryHandler &operator=(const ReplicationQueryHandler &) = default;
ReplicationQueryHandler(ReplicationQueryHandler &&) = delete;
ReplicationQueryHandler &operator=(ReplicationQueryHandler &&) = delete;
ReplicationQueryHandler(ReplicationQueryHandler &&) = default;
ReplicationQueryHandler &operator=(ReplicationQueryHandler &&) = default;
struct Replica {
std::string name;
@ -139,64 +140,6 @@ struct PreparedQuery {
plan::ReadWriteTypeChecker::RWType rw_type;
};
// TODO: Maybe this should move to query/plan/planner.
/// Interface for accessing the root operator of a logical plan.
class LogicalPlan {
public:
virtual ~LogicalPlan() {}
virtual const plan::LogicalOperator &GetRoot() const = 0;
virtual double GetCost() const = 0;
virtual const SymbolTable &GetSymbolTable() const = 0;
virtual const AstStorage &GetAstStorage() const = 0;
};
class CachedPlan {
public:
explicit CachedPlan(std::unique_ptr<LogicalPlan> plan);
const auto &plan() const { return plan_->GetRoot(); }
double cost() const { return plan_->GetCost(); }
const auto &symbol_table() const { return plan_->GetSymbolTable(); }
const auto &ast_storage() const { return plan_->GetAstStorage(); }
bool IsExpired() const { return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_plan_cache_ttl); };
private:
std::unique_ptr<LogicalPlan> plan_;
utils::Timer cache_timer_;
};
struct CachedQuery {
AstStorage ast_storage;
Query *query;
std::vector<AuthQuery::Privilege> required_privileges;
};
struct QueryCacheEntry {
bool operator==(const QueryCacheEntry &other) const { return first == other.first; }
bool operator<(const QueryCacheEntry &other) const { return first < other.first; }
bool operator==(const uint64_t &other) const { return first == other; }
bool operator<(const uint64_t &other) const { return first < other; }
uint64_t first;
// TODO: Maybe store the query string here and use it as a key with the hash
// so that we eliminate the risk of hash collisions.
CachedQuery second;
};
struct PlanCacheEntry {
bool operator==(const PlanCacheEntry &other) const { return first == other.first; }
bool operator<(const PlanCacheEntry &other) const { return first < other.first; }
bool operator==(const uint64_t &other) const { return first == other; }
bool operator<(const uint64_t &other) const { return first < other; }
uint64_t first;
// TODO: Maybe store the query string here and use it as a key with the hash
// so that we eliminate the risk of hash collisions.
std::shared_ptr<CachedPlan> second;
};
/**
* Holds data shared between multiple `Interpreter` instances (which might be
* running concurrently).
@ -205,7 +148,7 @@ struct PlanCacheEntry {
* been passed to an `Interpreter` instance.
*/
struct InterpreterContext {
explicit InterpreterContext(storage::Storage *db) : db(db) {}
explicit InterpreterContext(storage::Storage *db, const std::filesystem::path &data_directory);
storage::Storage *db;
@ -225,6 +168,9 @@ struct InterpreterContext {
utils::SkipList<QueryCacheEntry> ast_cache;
utils::SkipList<PlanCacheEntry> plan_cache;
std::optional<TriggerStore> trigger_store;
utils::ThreadPool after_commit_trigger_pool{1};
};
/// Function that is used to tell all active interpreters that they should stop
@ -309,6 +255,9 @@ class Interpreter final {
void RollbackTransaction();
void SetNextTransactionIsolationLevel(storage::IsolationLevel isolation_level);
void SetSessionIsolationLevel(storage::IsolationLevel isolation_level);
/**
* Abort the current multicommand transaction.
*/
@ -317,7 +266,9 @@ class Interpreter final {
private:
struct QueryExecution {
std::optional<PreparedQuery> prepared_query;
utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
utils::MonotonicBufferResource execution_monotonic_memory{kExecutionMemoryBlockSize};
utils::ResourceWithOutOfMemoryException execution_memory{&execution_monotonic_memory};
std::map<std::string, TypedValue> summary;
explicit QueryExecution() = default;
@ -331,7 +282,7 @@ class Interpreter final {
// destroy the prepared query which is using that instance
// of execution memory.
prepared_query.reset();
execution_memory.Release();
execution_monotonic_memory.Release();
}
};
@ -350,15 +301,23 @@ class Interpreter final {
InterpreterContext *interpreter_context_;
std::optional<storage::Storage::Accessor> db_accessor_;
// This cannot be std::optional because we need to move this accessor later on into a lambda capture
// which is assigned to std::function. std::function requires every object to be copyable, so we
// move this unique_ptr into a shrared_ptr.
std::unique_ptr<storage::Storage::Accessor> db_accessor_;
std::optional<DbAccessor> execution_db_accessor_;
std::optional<TriggerContextCollector> trigger_context_collector_;
bool in_explicit_transaction_{false};
bool expect_rollback_{false};
std::optional<storage::IsolationLevel> interpreter_isolation_level;
std::optional<storage::IsolationLevel> next_transaction_isolation_level;
PreparedQuery PrepareTransactionQuery(std::string_view query_upper);
void Commit();
void AdvanceCommand();
void AbortCommand(std::unique_ptr<QueryExecution> *query_execution);
std::optional<storage::IsolationLevel> GetIsolationLevelOverride();
size_t ActiveQueryExecutions() {
return std::count_if(query_executions_.begin(), query_executions_.end(),

377
src/query/plan/operator.cpp
View File

@ -15,6 +15,7 @@
#include <cppitertools/imap.hpp>
#include "query/context.hpp"
#include "query/db_accessor.hpp"
#include "query/exceptions.hpp"
#include "query/frontend/ast/ast.hpp"
#include "query/frontend/semantic/symbol_table.hpp"
@ -23,14 +24,18 @@
#include "query/plan/scoped_profile.hpp"
#include "query/procedure/mg_procedure_impl.hpp"
#include "query/procedure/module.hpp"
#include "storage/v2/property_value.hpp"
#include "utils/algorithm.hpp"
#include "utils/csv_parsing.hpp"
#include "utils/event_counter.hpp"
#include "utils/exceptions.hpp"
#include "utils/fnv.hpp"
#include "utils/likely.hpp"
#include "utils/logging.hpp"
#include "utils/pmr/unordered_map.hpp"
#include "utils/pmr/unordered_set.hpp"
#include "utils/pmr/vector.hpp"
#include "utils/readable_size.hpp"
#include "utils/string.hpp"
// macro for the default implementation of LogicalOperator::Accept
@ -203,7 +208,10 @@ bool CreateNode::CreateNodeCursor::Pull(Frame &frame, ExecutionContext &context)
SCOPED_PROFILE_OP("CreateNode");
if (input_cursor_->Pull(frame, context)) {
CreateLocalVertex(self_.node_info_, &frame, context);
auto created_vertex = CreateLocalVertex(self_.node_info_, &frame, context);
if (context.trigger_context_collector) {
context.trigger_context_collector->RegisterCreatedObject(created_vertex);
}
return true;
}
@ -242,8 +250,8 @@ CreateExpand::CreateExpandCursor::CreateExpandCursor(const CreateExpand &self, u
namespace {
void CreateEdge(const EdgeCreationInfo &edge_info, DbAccessor *dba, VertexAccessor *from, VertexAccessor *to,
Frame *frame, ExpressionEvaluator *evaluator) {
EdgeAccessor CreateEdge(const EdgeCreationInfo &edge_info, DbAccessor *dba, VertexAccessor *from, VertexAccessor *to,
Frame *frame, ExpressionEvaluator *evaluator) {
auto maybe_edge = dba->InsertEdge(from, to, edge_info.edge_type);
if (maybe_edge.HasValue()) {
auto &edge = *maybe_edge;
@ -261,6 +269,8 @@ void CreateEdge(const EdgeCreationInfo &edge_info, DbAccessor *dba, VertexAccess
throw QueryRuntimeException("Unexpected error when creating an edge.");
}
}
return *maybe_edge;
}
} // namespace
@ -286,19 +296,23 @@ bool CreateExpand::CreateExpandCursor::Pull(Frame &frame, ExecutionContext &cont
// create an edge between the two nodes
auto *dba = context.db_accessor;
switch (self_.edge_info_.direction) {
case EdgeAtom::Direction::IN:
CreateEdge(self_.edge_info_, dba, &v2, &v1, &frame, &evaluator);
break;
case EdgeAtom::Direction::OUT:
CreateEdge(self_.edge_info_, dba, &v1, &v2, &frame, &evaluator);
break;
case EdgeAtom::Direction::BOTH:
auto created_edge = [&] {
switch (self_.edge_info_.direction) {
case EdgeAtom::Direction::IN:
return CreateEdge(self_.edge_info_, dba, &v2, &v1, &frame, &evaluator);
case EdgeAtom::Direction::OUT:
// in the case of an undirected CreateExpand we choose an arbitrary
// direction. this is used in the MERGE clause
// it is not allowed in the CREATE clause, and the semantic
// checker needs to ensure it doesn't reach this point
CreateEdge(self_.edge_info_, dba, &v1, &v2, &frame, &evaluator);
case EdgeAtom::Direction::BOTH:
return CreateEdge(self_.edge_info_, dba, &v1, &v2, &frame, &evaluator);
}
}();
if (context.trigger_context_collector) {
context.trigger_context_collector->RegisterCreatedObject(created_edge);
}
return true;
@ -314,18 +328,26 @@ VertexAccessor &CreateExpand::CreateExpandCursor::OtherVertex(Frame &frame, Exec
ExpectType(self_.node_info_.symbol, dest_node_value, TypedValue::Type::Vertex);
return dest_node_value.ValueVertex();
} else {
return CreateLocalVertex(self_.node_info_, &frame, context);
auto &created_vertex = CreateLocalVertex(self_.node_info_, &frame, context);
if (context.trigger_context_collector) {
context.trigger_context_collector->RegisterCreatedObject(created_vertex);
}
return created_vertex;
}
}
template <class TVerticesFun>
class ScanAllCursor : public Cursor {
public:
explicit ScanAllCursor(Symbol output_symbol, UniqueCursorPtr input_cursor, TVerticesFun get_vertices)
: output_symbol_(output_symbol), input_cursor_(std::move(input_cursor)), get_vertices_(std::move(get_vertices)) {}
explicit ScanAllCursor(Symbol output_symbol, UniqueCursorPtr input_cursor, TVerticesFun get_vertices,
const char *op_name)
: output_symbol_(output_symbol),
input_cursor_(std::move(input_cursor)),
get_vertices_(std::move(get_vertices)),
op_name_(op_name) {}
bool Pull(Frame &frame, ExecutionContext &context) override {
SCOPED_PROFILE_OP("ScanAll");
SCOPED_PROFILE_OP(op_name_);
if (MustAbort(context)) throw HintedAbortError();
@ -361,6 +383,7 @@ class ScanAllCursor : public Cursor {
TVerticesFun get_vertices_;
std::optional<typename std::result_of<TVerticesFun(Frame &, ExecutionContext &)>::type::value_type> vertices_;
std::optional<decltype(vertices_.value().begin())> vertices_it_;
const char *op_name_;
};
ScanAll::ScanAll(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol, storage::View view)
@ -376,7 +399,7 @@ UniqueCursorPtr ScanAll::MakeCursor(utils::MemoryResource *mem) const {
return std::make_optional(db->Vertices(view_));
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices));
std::move(vertices), "ScanAll");
}
std::vector<Symbol> ScanAll::ModifiedSymbols(const SymbolTable &table) const {
@ -399,7 +422,7 @@ UniqueCursorPtr ScanAllByLabel::MakeCursor(utils::MemoryResource *mem) const {
return std::make_optional(db->Vertices(view_, label_));
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices));
std::move(vertices), "ScanAllByLabel");
}
// TODO(buda): Implement ScanAllByLabelProperty operator to iterate over
@ -463,7 +486,7 @@ UniqueCursorPtr ScanAllByLabelPropertyRange::MakeCursor(utils::MemoryResource *m
return std::make_optional(db->Vertices(view_, label_, property_, maybe_lower, maybe_upper));
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices));
std::move(vertices), "ScanAllByLabelPropertyRange");
}
ScanAllByLabelPropertyValue::ScanAllByLabelPropertyValue(const std::shared_ptr<LogicalOperator> &input,
@ -495,7 +518,7 @@ UniqueCursorPtr ScanAllByLabelPropertyValue::MakeCursor(utils::MemoryResource *m
return std::make_optional(db->Vertices(view_, label_, property_, storage::PropertyValue(value)));
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices));
std::move(vertices), "ScanAllByLabelPropertyValue");
}
ScanAllByLabelProperty::ScanAllByLabelProperty(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol,
@ -513,7 +536,7 @@ UniqueCursorPtr ScanAllByLabelProperty::MakeCursor(utils::MemoryResource *mem) c
return std::make_optional(db->Vertices(view_, label_, property_));
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices));
std::move(vertices), "ScanAllByLabelProperty");
}
ScanAllById::ScanAllById(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol, Expression *expression,
@ -539,7 +562,7 @@ UniqueCursorPtr ScanAllById::MakeCursor(utils::MemoryResource *mem) const {
return std::vector<VertexAccessor>{*maybe_vertex};
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices));
std::move(vertices), "ScanAllById");
}
namespace {
@ -1794,8 +1817,7 @@ bool Delete::DeleteCursor::Pull(Frame &frame, ExecutionContext &context) {
if (!input_cursor_->Pull(frame, context)) return false;
// Delete should get the latest information, this way it is also possible
// to
// delete newly added nodes and edges.
// to delete newly added nodes and edges.
ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor,
storage::View::NEW);
auto *pull_memory = context.evaluation_context.memory;
@ -1813,9 +1835,9 @@ bool Delete::DeleteCursor::Pull(Frame &frame, ExecutionContext &context) {
for (TypedValue &expression_result : expression_results) {
if (MustAbort(context)) throw HintedAbortError();
if (expression_result.type() == TypedValue::Type::Edge) {
auto maybe_error = dba.RemoveEdge(&expression_result.ValueEdge());
if (maybe_error.HasError()) {
switch (maybe_error.GetError()) {
auto maybe_value = dba.RemoveEdge(&expression_result.ValueEdge());
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case storage::Error::SERIALIZATION_ERROR:
throw QueryRuntimeException("Can't serialize due to concurrent operations.");
case storage::Error::DELETED_OBJECT:
@ -1825,6 +1847,10 @@ bool Delete::DeleteCursor::Pull(Frame &frame, ExecutionContext &context) {
throw QueryRuntimeException("Unexpected error when deleting an edge.");
}
}
if (context.trigger_context_collector && maybe_value.GetValue()) {
context.trigger_context_collector->RegisterDeletedObject(*maybe_value.GetValue());
}
}
}
@ -1835,9 +1861,9 @@ bool Delete::DeleteCursor::Pull(Frame &frame, ExecutionContext &context) {
case TypedValue::Type::Vertex: {
auto &va = expression_result.ValueVertex();
if (self_.detach_) {
auto maybe_error = dba.DetachRemoveVertex(&va);
if (maybe_error.HasError()) {
switch (maybe_error.GetError()) {
auto res = dba.DetachRemoveVertex(&va);
if (res.HasError()) {
switch (res.GetError()) {
case storage::Error::SERIALIZATION_ERROR:
throw QueryRuntimeException("Can't serialize due to concurrent operations.");
case storage::Error::DELETED_OBJECT:
@ -1847,6 +1873,13 @@ bool Delete::DeleteCursor::Pull(Frame &frame, ExecutionContext &context) {
throw QueryRuntimeException("Unexpected error when deleting a node.");
}
}
if (context.trigger_context_collector &&
context.trigger_context_collector->ShouldRegisterDeletedObject<EdgeAccessor>() && res.GetValue()) {
context.trigger_context_collector->RegisterDeletedObject(res.GetValue()->first);
for (const auto &deleted_edge : res.GetValue()->second) {
context.trigger_context_collector->RegisterDeletedObject(deleted_edge);
}
}
} else {
auto res = dba.RemoveVertex(&va);
if (res.HasError()) {
@ -1861,6 +1894,10 @@ bool Delete::DeleteCursor::Pull(Frame &frame, ExecutionContext &context) {
throw QueryRuntimeException("Unexpected error when deleting a node.");
}
}
if (context.trigger_context_collector && res.GetValue()) {
context.trigger_context_collector->RegisterDeletedObject(*res.GetValue());
}
}
break;
}
@ -1914,12 +1951,26 @@ bool SetProperty::SetPropertyCursor::Pull(Frame &frame, ExecutionContext &contex
TypedValue rhs = self_.rhs_->Accept(evaluator);
switch (lhs.type()) {
case TypedValue::Type::Vertex:
PropsSetChecked(&lhs.ValueVertex(), self_.property_, rhs);
case TypedValue::Type::Vertex: {
auto old_value = PropsSetChecked(&lhs.ValueVertex(), self_.property_, rhs);
if (context.trigger_context_collector) {
// rhs cannot be moved because it was created with the allocator that is only valid during current pull
context.trigger_context_collector->RegisterSetObjectProperty(lhs.ValueVertex(), self_.property_,
TypedValue{std::move(old_value)}, TypedValue{rhs});
}
break;
case TypedValue::Type::Edge:
PropsSetChecked(&lhs.ValueEdge(), self_.property_, rhs);
}
case TypedValue::Type::Edge: {
auto old_value = PropsSetChecked(&lhs.ValueEdge(), self_.property_, rhs);
if (context.trigger_context_collector) {
// rhs cannot be moved because it was created with the allocator that is only valid during current pull
context.trigger_context_collector->RegisterSetObjectProperty(lhs.ValueEdge(), self_.property_,
TypedValue{std::move(old_value)}, TypedValue{rhs});
}
break;
}
case TypedValue::Type::Null:
// Skip setting properties on Null (can occur in optional match).
break;
@ -1959,16 +2010,29 @@ SetProperties::SetPropertiesCursor::SetPropertiesCursor(const SetProperties &sel
namespace {
template <typename T>
concept AccessorWithProperties = requires(T value, storage::PropertyId property_id,
storage::PropertyValue property_value) {
{ value.ClearProperties() }
->std::same_as<storage::Result<std::map<storage::PropertyId, storage::PropertyValue>>>;
{value.SetProperty(property_id, property_value)};
};
/// Helper function that sets the given values on either a Vertex or an Edge.
///
/// @tparam TRecordAccessor Either RecordAccessor<Vertex> or
/// RecordAccessor<Edge>
template <typename TRecordAccessor>
void SetPropertiesOnRecord(DbAccessor *dba, TRecordAccessor *record, const TypedValue &rhs, SetProperties::Op op) {
template <AccessorWithProperties TRecordAccessor>
void SetPropertiesOnRecord(TRecordAccessor *record, const TypedValue &rhs, SetProperties::Op op,
ExecutionContext *context) {
std::optional<std::map<storage::PropertyId, storage::PropertyValue>> old_values;
const bool should_register_change =
context->trigger_context_collector &&
context->trigger_context_collector->ShouldRegisterObjectPropertyChange<TRecordAccessor>();
if (op == SetProperties::Op::REPLACE) {
auto maybe_error = record->ClearProperties();
if (maybe_error.HasError()) {
switch (maybe_error.GetError()) {
auto maybe_value = record->ClearProperties();
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case storage::Error::DELETED_OBJECT:
throw QueryRuntimeException("Trying to set properties on a deleted graph element.");
case storage::Error::SERIALIZATION_ERROR:
@ -1980,6 +2044,10 @@ void SetPropertiesOnRecord(DbAccessor *dba, TRecordAccessor *record, const Typed
throw QueryRuntimeException("Unexpected error when setting properties.");
}
}
if (should_register_change) {
old_values.emplace(std::move(*maybe_value));
}
}
auto get_props = [](const auto &record) {
@ -1999,8 +2067,25 @@ void SetPropertiesOnRecord(DbAccessor *dba, TRecordAccessor *record, const Typed
return *maybe_props;
};
auto set_props = [record](const auto &properties) {
for (const auto &kv : properties) {
auto register_set_property = [&](auto &&returned_old_value, auto key, auto &&new_value) {
auto old_value = [&]() -> storage::PropertyValue {
if (!old_values) {
return std::forward<decltype(returned_old_value)>(returned_old_value);
}
if (auto it = old_values->find(key); it != old_values->end()) {
return std::move(it->second);
}
return {};
}();
context->trigger_context_collector->RegisterSetObjectProperty(
*record, key, TypedValue(std::move(old_value)), TypedValue(std::forward<decltype(new_value)>(new_value)));
};
auto set_props = [&, record](auto properties) {
for (auto &kv : properties) {
auto maybe_error = record->SetProperty(kv.first, kv.second);
if (maybe_error.HasError()) {
switch (maybe_error.GetError()) {
@ -2015,6 +2100,10 @@ void SetPropertiesOnRecord(DbAccessor *dba, TRecordAccessor *record, const Typed
throw QueryRuntimeException("Unexpected error when setting properties.");
}
}
if (should_register_change) {
register_set_property(std::move(*maybe_error), kv.first, std::move(kv.second));
}
}
};
@ -2026,7 +2115,13 @@ void SetPropertiesOnRecord(DbAccessor *dba, TRecordAccessor *record, const Typed
set_props(get_props(rhs.ValueVertex()));
break;
case TypedValue::Type::Map: {
for (const auto &kv : rhs.ValueMap()) PropsSetChecked(record, dba->NameToProperty(kv.first), kv.second);
for (const auto &kv : rhs.ValueMap()) {
auto key = context->db_accessor->NameToProperty(kv.first);
auto old_value = PropsSetChecked(record, key, kv.second);
if (should_register_change) {
register_set_property(std::move(old_value), key, kv.second);
}
}
break;
}
default:
@ -2034,6 +2129,14 @@ void SetPropertiesOnRecord(DbAccessor *dba, TRecordAccessor *record, const Typed
"Right-hand side in SET expression must be a node, an edge or a "
"map.");
}
if (should_register_change && old_values) {
// register removed properties
for (auto &[property_id, property_value] : *old_values) {
context->trigger_context_collector->RegisterRemovedObjectProperty(*record, property_id,
TypedValue(std::move(property_value)));
}
}
}
} // namespace
@ -2052,10 +2155,10 @@ bool SetProperties::SetPropertiesCursor::Pull(Frame &frame, ExecutionContext &co
switch (lhs.type()) {
case TypedValue::Type::Vertex:
SetPropertiesOnRecord(context.db_accessor, &lhs.ValueVertex(), rhs, self_.op_);
SetPropertiesOnRecord(&lhs.ValueVertex(), rhs, self_.op_, &context);
break;
case TypedValue::Type::Edge:
SetPropertiesOnRecord(context.db_accessor, &lhs.ValueEdge(), rhs, self_.op_);
SetPropertiesOnRecord(&lhs.ValueEdge(), rhs, self_.op_, &context);
break;
case TypedValue::Type::Null:
// Skip setting properties on Null (can occur in optional match).
@ -2100,9 +2203,9 @@ bool SetLabels::SetLabelsCursor::Pull(Frame &frame, ExecutionContext &context) {
ExpectType(self_.input_symbol_, vertex_value, TypedValue::Type::Vertex);
auto &vertex = vertex_value.ValueVertex();
for (auto label : self_.labels_) {
auto maybe_error = vertex.AddLabel(label);
if (maybe_error.HasError()) {
switch (maybe_error.GetError()) {
auto maybe_value = vertex.AddLabel(label);
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case storage::Error::SERIALIZATION_ERROR:
throw QueryRuntimeException("Can't serialize due to concurrent operations.");
case storage::Error::DELETED_OBJECT:
@ -2113,6 +2216,10 @@ bool SetLabels::SetLabelsCursor::Pull(Frame &frame, ExecutionContext &context) {
throw QueryRuntimeException("Unexpected error when setting a label.");
}
}
if (context.trigger_context_collector && *maybe_value) {
context.trigger_context_collector->RegisterSetVertexLabel(vertex, label);
}
}
return true;
@ -2151,10 +2258,10 @@ bool RemoveProperty::RemovePropertyCursor::Pull(Frame &frame, ExecutionContext &
storage::View::NEW);
TypedValue lhs = self_.lhs_->expression_->Accept(evaluator);
auto remove_prop = [property = self_.property_](auto *record) {
auto maybe_error = record->RemoveProperty(property);
if (maybe_error.HasError()) {
switch (maybe_error.GetError()) {
auto remove_prop = [property = self_.property_, &context](auto *record) {
auto maybe_old_value = record->RemoveProperty(property);
if (maybe_old_value.HasError()) {
switch (maybe_old_value.GetError()) {
case storage::Error::DELETED_OBJECT:
throw QueryRuntimeException("Trying to remove a property on a deleted graph element.");
case storage::Error::SERIALIZATION_ERROR:
@ -2168,6 +2275,11 @@ bool RemoveProperty::RemovePropertyCursor::Pull(Frame &frame, ExecutionContext &
throw QueryRuntimeException("Unexpected error when removing property.");
}
}
if (context.trigger_context_collector) {
context.trigger_context_collector->RegisterRemovedObjectProperty(*record, property,
TypedValue(std::move(*maybe_old_value)));
}
};
switch (lhs.type()) {
@ -2220,9 +2332,9 @@ bool RemoveLabels::RemoveLabelsCursor::Pull(Frame &frame, ExecutionContext &cont
ExpectType(self_.input_symbol_, vertex_value, TypedValue::Type::Vertex);
auto &vertex = vertex_value.ValueVertex();
for (auto label : self_.labels_) {
auto maybe_error = vertex.RemoveLabel(label);
if (maybe_error.HasError()) {
switch (maybe_error.GetError()) {
auto maybe_value = vertex.RemoveLabel(label);
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case storage::Error::SERIALIZATION_ERROR:
throw QueryRuntimeException("Can't serialize due to concurrent operations.");
case storage::Error::DELETED_OBJECT:
@ -2233,6 +2345,10 @@ bool RemoveLabels::RemoveLabelsCursor::Pull(Frame &frame, ExecutionContext &cont
throw QueryRuntimeException("Unexpected error when removing labels from a node.");
}
}
if (context.trigger_context_collector && *maybe_value) {
context.trigger_context_collector->RegisterRemovedVertexLabel(vertex, label);
}
}
return true;
@ -3485,16 +3601,6 @@ std::unordered_map<std::string, int64_t> CallProcedure::GetAndResetCounters() {
namespace {
std::optional<size_t> EvalMemoryLimit(ExpressionEvaluator *eval, Expression *memory_limit, size_t memory_scale) {
if (!memory_limit) return std::nullopt;
auto limit_value = memory_limit->Accept(*eval);
if (!limit_value.IsInt() || limit_value.ValueInt() <= 0)
throw QueryRuntimeException("Memory limit must be a non-negative integer.");
size_t limit = limit_value.ValueInt();
if (std::numeric_limits<size_t>::max() / memory_scale < limit) throw QueryRuntimeException("Memory limit overflow.");
return limit * memory_scale;
}
void CallCustomProcedure(const std::string_view &fully_qualified_procedure_name, const mgp_proc &proc,
const std::vector<Expression *> &args, const mgp_graph &graph, ExpressionEvaluator *evaluator,
utils::MemoryResource *memory, std::optional<size_t> memory_limit, mgp_result *result) {
@ -3544,7 +3650,8 @@ void CallCustomProcedure(const std::string_view &fully_qualified_procedure_name,
proc_args.elems.emplace_back(std::get<2>(proc.opt_args[i]), &graph);
}
if (memory_limit) {
SPDLOG_INFO("Running '{}' with memory limit of {} bytes", fully_qualified_procedure_name, *memory_limit);
SPDLOG_INFO("Running '{}' with memory limit of {}", fully_qualified_procedure_name,
utils::GetReadableSize(*memory_limit));
utils::LimitedMemoryResource limited_mem(memory, *memory_limit);
mgp_memory proc_memory{&limited_mem};
MG_ASSERT(result->signature == &proc.results);
@ -3623,7 +3730,7 @@ class CallProcedureCursor : public Cursor {
// TODO: This will probably need to be changed when we add support for
// generator like procedures which yield a new result on each invocation.
auto *memory = context.evaluation_context.memory;
auto memory_limit = EvalMemoryLimit(&evaluator, self_->memory_limit_, self_->memory_scale_);
auto memory_limit = EvaluateMemoryLimit(&evaluator, self_->memory_limit_, self_->memory_scale_);
mgp_graph graph{context.db_accessor, graph_view, &context};
CallCustomProcedure(self_->procedure_name_, *proc, self_->arguments_, graph, &evaluator, memory, memory_limit,
&result_);
@ -3679,4 +3786,142 @@ UniqueCursorPtr CallProcedure::MakeCursor(utils::MemoryResource *mem) const {
return MakeUniqueCursorPtr<CallProcedureCursor>(mem, this, mem);
}
LoadCsv::LoadCsv(std::shared_ptr<LogicalOperator> input, Expression *file, bool with_header, bool ignore_bad,
Expression *delimiter, Expression *quote, Symbol row_var)
: input_(input ? input : (std::make_shared<Once>())),
file_(file),
with_header_(with_header),
ignore_bad_(ignore_bad),
delimiter_(delimiter),
quote_(quote),
row_var_(row_var) {
MG_ASSERT(file_, "Something went wrong - '{}' member file_ shouldn't be a nullptr", __func__);
}
bool LoadCsv::Accept(HierarchicalLogicalOperatorVisitor &visitor) { return false; };
class LoadCsvCursor;
std::vector<Symbol> LoadCsv::OutputSymbols(const SymbolTable &sym_table) const { return {row_var_}; };
std::vector<Symbol> LoadCsv::ModifiedSymbols(const SymbolTable &sym_table) const {
auto symbols = input_->ModifiedSymbols(sym_table);
symbols.push_back(row_var_);
return symbols;
};
namespace {
// copy-pasted from interpreter.cpp
TypedValue EvaluateOptionalExpression(Expression *expression, ExpressionEvaluator *eval) {
return expression ? expression->Accept(*eval) : TypedValue();
}
auto ToOptionalString(ExpressionEvaluator *evaluator, Expression *expression) -> std::optional<utils::pmr::string> {
const auto evaluated_expr = EvaluateOptionalExpression(expression, evaluator);
if (evaluated_expr.IsString()) {
return utils::pmr::string(evaluated_expr.ValueString(), utils::NewDeleteResource());
}
return std::nullopt;
};
TypedValue CsvRowToTypedList(csv::Reader::Row row) {
auto *mem = row.get_allocator().GetMemoryResource();
auto typed_columns = utils::pmr::vector<TypedValue>(mem);
typed_columns.reserve(row.size());
for (auto &column : row) {
typed_columns.emplace_back(std::move(column));
}
return TypedValue(typed_columns, mem);
}
TypedValue CsvRowToTypedMap(csv::Reader::Row row, csv::Reader::Header header) {
// a valid row has the same number of elements as the header
auto *mem = row.get_allocator().GetMemoryResource();
utils::pmr::map<utils::pmr::string, TypedValue> m(mem);
for (auto i = 0; i < row.size(); ++i) {
m.emplace(std::move(header[i]), std::move(row[i]));
}
return TypedValue(m, mem);
}
} // namespace
class LoadCsvCursor : public Cursor {
const LoadCsv *self_;
const UniqueCursorPtr input_cursor_;
bool input_is_once_;
std::optional<csv::Reader> reader_{};
public:
LoadCsvCursor(const LoadCsv *self, utils::MemoryResource *mem)
: self_(self), input_cursor_(self_->input_->MakeCursor(mem)) {
input_is_once_ = dynamic_cast<Once *>(self_->input_.get());
}
bool Pull(Frame &frame, ExecutionContext &context) override {
SCOPED_PROFILE_OP("LoadCsv");
if (MustAbort(context)) throw HintedAbortError();
// ToDo(the-joksim):
// - this is an ungodly hack because the pipeline of creating a plan
// doesn't allow evaluating the expressions contained in self_->file_,
// self_->delimiter_, and self_->quote_ earlier (say, in the interpreter.cpp)
// without massacring the code even worse than I did here
if (UNLIKELY(!reader_)) {
reader_ = MakeReader(&context.evaluation_context);
}
bool input_pulled = input_cursor_->Pull(frame, context);
// If the input is Once, we have to keep going until we read all the rows,
// regardless of whether the pull on Once returned false.
// If we have e.g. MATCH(n) LOAD CSV ... AS x SET n.name = x.name, then we
// have to read at most cardinality(n) rows (but we can read less and stop
// pulling MATCH).
if (!input_is_once_ && !input_pulled) return false;
if (auto row = reader_->GetNextRow(context.evaluation_context.memory)) {
if (!reader_->HasHeader()) {
frame[self_->row_var_] = CsvRowToTypedList(std::move(*row));
} else {
frame[self_->row_var_] = CsvRowToTypedMap(
std::move(*row), csv::Reader::Header(reader_->GetHeader(), context.evaluation_context.memory));
}
return true;
}
return false;
}
void Reset() override { input_cursor_->Reset(); }
void Shutdown() override { input_cursor_->Shutdown(); }
private:
csv::Reader MakeReader(EvaluationContext *eval_context) {
Frame frame(0);
SymbolTable symbol_table;
DbAccessor *dba = nullptr;
auto evaluator = ExpressionEvaluator(&frame, symbol_table, *eval_context, dba, storage::View::OLD);
auto maybe_file = ToOptionalString(&evaluator, self_->file_);
auto maybe_delim = ToOptionalString(&evaluator, self_->delimiter_);
auto maybe_quote = ToOptionalString(&evaluator, self_->quote_);
// No need to check if maybe_file is std::nullopt, as the parser makes sure
// we can't get a nullptr for the 'file_' member in the LoadCsv clause.
// Note that the reader has to be given its own memory resource, as it
// persists between pulls, so it can't use the evalutation context memory
// resource.
return csv::Reader(
*maybe_file,
csv::Reader::Config(self_->with_header_, self_->ignore_bad_, std::move(maybe_delim), std::move(maybe_quote)),
utils::NewDeleteResource());
}
};
UniqueCursorPtr LoadCsv::MakeCursor(utils::MemoryResource *mem) const {
return MakeUniqueCursorPtr<LoadCsvCursor>(mem, this, mem);
};
} // namespace query::plan

36
src/query/plan/operator.lcp
View File

@ -117,6 +117,7 @@ class Distinct;
class Union;
class Cartesian;
class CallProcedure;
class LoadCsv;
using LogicalOperatorCompositeVisitor = ::utils::CompositeVisitor<
Once, CreateNode, CreateExpand, ScanAll, ScanAllByLabel,
@ -125,7 +126,7 @@ using LogicalOperatorCompositeVisitor = ::utils::CompositeVisitor<
Expand, ExpandVariable, ConstructNamedPath, Filter, Produce, Delete,
SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels,
EdgeUniquenessFilter, Accumulate, Aggregate, Skip, Limit, OrderBy, Merge,
Optional, Unwind, Distinct, Union, Cartesian, CallProcedure>;
Optional, Unwind, Distinct, Union, Cartesian, CallProcedure, LoadCsv>;
using LogicalOperatorLeafVisitor = ::utils::LeafVisitor<Once>;
@ -2156,5 +2157,38 @@ at once. Instead, each call of the callback should return a single row of the ta
(:serialize (:slk))
(:clone))
(lcp:define-class load-csv (logical-operator)
((input "std::shared_ptr<LogicalOperator>" :scope :public
:slk-save #'slk-save-operator-pointer
:slk-load #'slk-load-operator-pointer)
(file "Expression *" :scope :public)
(with_header "bool" :scope :public)
(ignore_bad "bool" :scope :public)
(delimiter "Expression *" :initval "nullptr" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression"))
(quote "Expression *" :initval "nullptr" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression"))
(row_var "Symbol" :scope :public))
(:public
#>cpp
LoadCsv() = default;
LoadCsv(std::shared_ptr<LogicalOperator> input, Expression *file, bool with_header, bool ignore_bad,
Expression* delimiter, Expression* quote, Symbol row_var);
bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override;
UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override;
std::vector<Symbol> OutputSymbols(const SymbolTable &) const override;
std::vector<Symbol> ModifiedSymbols(const SymbolTable &) const override;
bool HasSingleInput() const override { return true; }
std::shared_ptr<LogicalOperator> input() const override { return input_; }
void set_input(std::shared_ptr<LogicalOperator> input) override {
input_ = input;
}
cpp<#)
(:serialize (:slk))
(:clone))
(lcp:pop-namespace) ;; plan
(lcp:pop-namespace) ;; query

3
src/query/plan/preprocess.cpp
View File

@ -522,7 +522,8 @@ std::vector<SingleQueryPart> CollectSingleQueryParts(SymbolTable &symbol_table,
query_part->merge_matching.emplace_back(Matching{});
AddMatching({merge->pattern_}, nullptr, symbol_table, storage, query_part->merge_matching.back());
} else if (utils::IsSubtype(*clause, With::kType) || utils::IsSubtype(*clause, query::Unwind::kType) ||
utils::IsSubtype(*clause, query::CallProcedure::kType)) {
utils::IsSubtype(*clause, query::CallProcedure::kType) ||
utils::IsSubtype(*clause, query::LoadCsv::kType)) {
// This query part is done, continue with a new one.
query_parts.emplace_back(SingleQueryPart{});
query_part = &query_parts.back();

22
src/query/plan/pretty_print.cpp
View File

@ -206,6 +206,11 @@ bool PlanPrinter::PreVisit(query::plan::CallProcedure &op) {
return true;
}
bool PlanPrinter::PreVisit(query::plan::LoadCsv &op) {
WithPrintLn([&op](auto &out) { out << "* LoadCsv {" << op.row_var_.name() << "}"; });
return true;
}
bool PlanPrinter::Visit(query::plan::Once &op) {
WithPrintLn([](auto &out) { out << "* Once"; });
return true;
@ -803,6 +808,23 @@ bool PlanToJsonVisitor::PreVisit(query::plan::CallProcedure &op) {
return false;
}
bool PlanToJsonVisitor::PreVisit(query::plan::LoadCsv &op) {
json self;
self["name"] = "LoadCsv";
self["file"] = ToJson(op.file_);
self["with_header"] = op.with_header_;
self["ignore_bad"] = op.ignore_bad_;
self["delimiter"] = ToJson(op.delimiter_);
self["quote"] = ToJson(op.quote_);
self["row_variable"] = ToJson(op.row_var_);
op.input_->Accept(*this);
self["input"] = PopOutput();
output_ = std::move(self);
return false;
}
bool PlanToJsonVisitor::PreVisit(Distinct &op) {
json self;
self["name"] = "Distinct";

2
src/query/plan/pretty_print.hpp
View File

@ -81,6 +81,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
bool PreVisit(Unwind &) override;
bool PreVisit(CallProcedure &) override;
bool PreVisit(LoadCsv &) override;
bool Visit(Once &) override;
@ -194,6 +195,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
bool PreVisit(Unwind &) override;
bool PreVisit(CallProcedure &) override;
bool PreVisit(LoadCsv &) override;
bool Visit(Once &) override;

13
src/query/plan/profile.cpp
View File

@ -98,10 +98,9 @@ class ProfilingStatsToTableHelper {
} // namespace
std::vector<std::vector<TypedValue>> ProfilingStatsToTable(const ProfilingStats &cumulative_stats,
std::chrono::duration<double> total_time) {
ProfilingStatsToTableHelper helper{cumulative_stats.num_cycles, total_time};
helper.Output(cumulative_stats);
std::vector<std::vector<TypedValue>> ProfilingStatsToTable(const ProfilingStatsWithTotalTime &stats) {
ProfilingStatsToTableHelper helper{stats.cumulative_stats.num_cycles, stats.total_time};
helper.Output(stats.cumulative_stats);
return helper.rows();
}
@ -147,9 +146,9 @@ class ProfilingStatsToJsonHelper {
} // namespace
nlohmann::json ProfilingStatsToJson(const ProfilingStats &cumulative_stats, std::chrono::duration<double> total_time) {
ProfilingStatsToJsonHelper helper{cumulative_stats.num_cycles, total_time};
helper.Output(cumulative_stats);
nlohmann::json ProfilingStatsToJson(const ProfilingStatsWithTotalTime &stats) {
ProfilingStatsToJsonHelper helper{stats.cumulative_stats.num_cycles, stats.total_time};
helper.Output(stats.cumulative_stats);
return helper.ToJson();
}

10
src/query/plan/profile.hpp
View File

@ -23,10 +23,14 @@ struct ProfilingStats {
std::vector<ProfilingStats> children;
};
std::vector<std::vector<TypedValue>> ProfilingStatsToTable(const ProfilingStats &cumulative_stats,
std::chrono::duration<double>);
struct ProfilingStatsWithTotalTime {
ProfilingStats cumulative_stats{};
std::chrono::duration<double> total_time{};
};
nlohmann::json ProfilingStatsToJson(const ProfilingStats &cumulative_stats, std::chrono::duration<double>);
std::vector<std::vector<TypedValue>> ProfilingStatsToTable(const ProfilingStatsWithTotalTime &stats);
nlohmann::json ProfilingStatsToJson(const ProfilingStatsWithTotalTime &stats);
} // namespace plan
} // namespace query

7
src/query/plan/rule_based_planner.hpp
View File

@ -203,6 +203,13 @@ class RuleBasedPlanner {
input_op = std::make_unique<plan::CallProcedure>(
std::move(input_op), call_proc->procedure_name_, call_proc->arguments_, call_proc->result_fields_,
result_symbols, call_proc->memory_limit_, call_proc->memory_scale_);
} else if (auto *load_csv = utils::Downcast<query::LoadCsv>(clause)) {
const auto &row_sym = context.symbol_table->at(*load_csv->row_var_);
context.bound_symbols.insert(row_sym);
input_op =
std::make_unique<plan::LoadCsv>(std::move(input_op), load_csv->file_, load_csv->with_header_,
load_csv->ignore_bad_, load_csv->delimiter_, load_csv->quote_, row_sym);
} else {
throw utils::NotYetImplemented("clause '{}' conversion to operator(s)", clause->GetTypeInfo().name);
}

37
src/query/procedure/mg_procedure_impl.cpp
View File

@ -6,21 +6,20 @@
#include <regex>
#include <type_traits>
#include "module.hpp"
#include "utils/algorithm.hpp"
#include "utils/logging.hpp"
#include "utils/math.hpp"
#include "utils/memory.hpp"
#include "utils/string.hpp"
// This file contains implementation of top level C API functions, but this is
// all actually part of query::procedure. So use that namespace for simplicity.
// NOLINTNEXTLINE(google-build-using-namespace)
using namespace query::procedure;
void *mgp_alloc(mgp_memory *memory, size_t size_in_bytes) {
return mgp_aligned_alloc(memory, size_in_bytes, alignof(std::max_align_t));
}
namespace {
void *mgp_aligned_alloc(mgp_memory *memory, const size_t size_in_bytes, const size_t alignment) {
void *MgpAlignedAllocImpl(utils::MemoryResource &memory, const size_t size_in_bytes, const size_t alignment) {
if (size_in_bytes == 0U || !utils::IsPow2(alignment)) return nullptr;
// Simplify alignment by always using values greater or equal to max_align.
const size_t alloc_align = std::max(alignment, alignof(std::max_align_t));
@ -37,7 +36,7 @@ void *mgp_aligned_alloc(mgp_memory *memory, const size_t size_in_bytes, const si
const size_t alloc_size = bytes_for_header + size_in_bytes;
if (alloc_size < size_in_bytes) return nullptr;
try {
void *ptr = memory->impl->Allocate(alloc_size, alloc_align);
void *ptr = memory.Allocate(alloc_size, alloc_align);
char *data = reinterpret_cast<char *>(ptr) + bytes_for_header;
std::memcpy(data - sizeof(size_in_bytes), &size_in_bytes, sizeof(size_in_bytes));
std::memcpy(data - sizeof(size_in_bytes) - sizeof(alloc_align), &alloc_align, sizeof(alloc_align));
@ -47,7 +46,7 @@ void *mgp_aligned_alloc(mgp_memory *memory, const size_t size_in_bytes, const si
}
}
void mgp_free(mgp_memory *memory, void *const p) {
void MgpFreeImpl(utils::MemoryResource &memory, void *const p) {
if (!p) return;
char *const data = reinterpret_cast<char *>(p);
// Read the header containing size & alignment info.
@ -63,9 +62,31 @@ void mgp_free(mgp_memory *memory, void *const p) {
const size_t alloc_size = bytes_for_header + size_in_bytes;
// Get the original ptr we allocated.
void *const original_ptr = data - bytes_for_header;
memory->impl->Deallocate(original_ptr, alloc_size, alloc_align);
memory.Deallocate(original_ptr, alloc_size, alloc_align);
}
} // namespace
void *mgp_alloc(mgp_memory *memory, size_t size_in_bytes) {
return mgp_aligned_alloc(memory, size_in_bytes, alignof(std::max_align_t));
}
void *mgp_aligned_alloc(mgp_memory *memory, const size_t size_in_bytes, const size_t alignment) {
return MgpAlignedAllocImpl(*memory->impl, size_in_bytes, alignment);
}
void mgp_free(mgp_memory *memory, void *const p) { MgpFreeImpl(*memory->impl, p); }
void *mgp_global_alloc(size_t size_in_bytes) {
return mgp_global_aligned_alloc(size_in_bytes, alignof(std::max_align_t));
}
void *mgp_global_aligned_alloc(size_t size_in_bytes, size_t alignment) {
return MgpAlignedAllocImpl(gModuleRegistry.GetSharedMemoryResource(), size_in_bytes, alignment);
}
void mgp_global_free(void *const p) { MgpFreeImpl(gModuleRegistry.GetSharedMemoryResource(), p); }
namespace {
// May throw whatever the constructor of U throws. `std::bad_alloc` is handled

3
src/query/procedure/module.cpp
View File

@ -1,4 +1,5 @@
#include "query/procedure/module.hpp"
#include "utils/memory.hpp"
extern "C" {
#include <dlfcn.h>
@ -478,6 +479,8 @@ void ModuleRegistry::UnloadAllModules() {
DoUnloadAllModules();
}
utils::MemoryResource &ModuleRegistry::GetSharedMemoryResource() { return *shared_; }
std::optional<std::pair<procedure::ModulePtr, const mgp_proc *>> FindProcedure(
const ModuleRegistry &module_registry, const std::string_view &fully_qualified_procedure_name,
utils::MemoryResource *memory) {

4
src/query/procedure/module.hpp
View File

@ -52,6 +52,7 @@ class ModulePtr final {
class ModuleRegistry final {
std::map<std::string, std::unique_ptr<Module>, std::less<>> modules_;
mutable utils::RWLock lock_{utils::RWLock::Priority::WRITE};
std::unique_ptr<utils::MemoryResource> shared_{std::make_unique<utils::ResourceWithOutOfMemoryException>()};
bool RegisterModule(const std::string_view &name, std::unique_ptr<Module> module);
@ -96,6 +97,9 @@ class ModuleRegistry final {
/// Takes a write lock.
void UnloadAllModules();
/// Returns the shared memory allocator used by modules
utils::MemoryResource &GetSharedMemoryResource();
private:
std::vector<std::filesystem::path> modules_dirs_;
};

2
src/query/procedure/py_module.cpp
View File

@ -505,7 +505,7 @@ std::optional<py::ExceptionInfo> AddMultipleRecordsFromPython(mgp_result *result
return std::nullopt;
}
void CallPythonProcedure(py::Object py_cb, const mgp_list *args, const mgp_graph *graph, mgp_result *result,
void CallPythonProcedure(const py::Object &py_cb, const mgp_list *args, const mgp_graph *graph, mgp_result *result,
mgp_memory *memory) {
auto gil = py::EnsureGIL();

Some files were not shown because too many files have changed in this diff Show More