Compare commits

...

312 Commits

Author SHA1 Message Date
jeremy
82957ef727 Impl of Scan with filters (unfinished) 2022-09-23 09:52:48 +02:00
jeremy
93460f0fee Merge branch 'T1010-MG-implement-query-engine-client' into add_distributed_unwind 2022-09-22 14:27:17 +02:00
Jeremy B
440571f721
Merge branch 'project-pineapples' into T1010-MG-implement-query-engine-client 2022-09-22 14:24:28 +02:00
jeremy
87b5c55c14 Implement DistributedUnwindCursor 2022-09-22 14:11:37 +02:00
jbajic
6d89a15956 Fix storage_v3_expr 2022-09-22 14:09:36 +02:00
Tyler Neely
ce788f5f65
Machine manager and shard stitch (#569) 2022-09-22 13:55:16 +02:00
kostas kyrimis
3fa3f06581 Merge branch 'project-pineapples' into T1010-MG-implement-query-engine-client 2022-09-21 18:22:07 +00:00
János Benjamin Antal
b8186bea2e
Lexicographically ordered storage (#454) 2022-09-21 18:58:31 +02:00
János Benjamin Antal
3c4856dcb7 Merge remote-tracking branch 'origin/project-pineapples' into E118-MG-lexicographically-ordered-storage 2022-09-21 18:50:00 +02:00
Marko Budiselić
b4d6dc0930
Add proper transaction handling (#550) 2022-09-21 18:25:51 +02:00
Jure Bajic
817161a915
Implement expression evaluator in storage v3 (#534)
- Enable `mg-expr` in mg-storage-v3
-  Adapt id_mapper
- Change conversion function from PropertyValue to TypedValue
- Remove memgraph functions
- Enable expression tests on for storage
2022-09-21 17:26:55 +02:00
kostas kyrimis
8d4c246d8f Address comments 2022-09-21 15:12:52 +00:00
Tyler Neely
58eb2caf0f Add machine manager prototype (#533) 2022-09-21 16:57:25 +02:00
kostas kyrimis
7ea0df2731 Make clang-tidy happy 2022-09-21 13:23:14 +00:00
gvolfing
b541548d49 Fix failing shard_rsm test 2022-09-21 15:01:37 +02:00
kostas kyrimis
f89af77be9 Merge branch 'E118-MG-lexicographically-ordered-storage' into T1010-MG-implement-query-engine-client 2022-09-21 11:35:30 +00:00
kostas kyrimis
834c80091e Remove unnecessary output on shard request manager test 2022-09-20 09:41:57 +00:00
gvolfing
ecda71168c
Implement message based actions (#538)
Create shard-side handlers for basic messages

Implement the handlers for CreateVertices, CreateEdges and ScanAll. Use
or modify the defined messages to interact with individual Shards and
test their behavior. Shard is currently being owned by ShardRsm
instances. The two top level dispatching functions Read() and Apply()
are responsible for read- and write operations respectively. Currently
there are a handful of messages that are defined but not utilized, these
will be used in the near future, as well as a couple of handler
functions with empty implementations.
2022-09-20 11:15:19 +02:00
kostas kyrimis
018800f01e Revert accidental changes on query/reequired_priviledges 2022-09-20 07:47:00 +00:00
kostas kyrimis
ba8cd4b492 Add dummy test to generate lisp deps so clang-tidy can work 2022-09-19 15:08:02 +00:00
kostas kyrimis
9c9d0343a9 Fix clang-tidy warnings 2022-09-19 13:55:25 +00:00
kostas kyrimis
708e745410 Clean up commented out code 2022-09-19 09:43:01 +00:00
Kostas Kyrimis
632f0398b5 Add missing glue code on DistributedCreateVertices 2022-09-14 15:12:16 +03:00
János Benjamin Antal
8e1f83acc9
Reference vertices by id in edges (#532) 2022-09-14 13:11:14 +02:00
Kostas Kyrimis
c182cf8384 Added prototype for distributed ScanAll and CreateVertices cursors.
Changed ScanAll request return type to be friendlier for use.
2022-09-12 16:54:35 +03:00
Kostas Kyrimis
b5f8060c1f Integrated new accessors with TypedValue and temporary disabled
procedures, streams, triggers. Additionally, small changes to
ExpressionEvaluator and commented out parts of the operators that
used old accessors
2022-09-09 20:56:24 +03:00
jbajic
f910cb770c Merge branch 'project-pineapples' into E118-MG-lexicographically-ordered-storage 2022-09-09 12:39:28 +02:00
Kostas Kyrimis
ced3b7db06 ExpandOne prototype 2022-09-09 12:48:04 +03:00
Kostas Kyrimis
e078947d10 Comment out query engine tests temporarily because of the new TypedValue 2022-09-08 19:57:48 +03:00
Kostas Kyrimis
2eaad15804 Added TypedValue with new accessors and simplified requests 2022-09-08 19:56:48 +03:00
Kostas Kyrimis
002e08b4f8 Merge branch 'project-pineapples' into T1010-MG-implement-query-engine-client 2022-09-07 18:17:46 +03:00
Kostas Kyrimis
38d0b89b04
Implement expression evaluator library (#486) 2022-09-07 18:15:32 +03:00
Kostas Kyrimis
e7d9ab1c5f Added accessors prototype, and ExpandOne request boilerplate 2022-09-07 17:36:49 +03:00
Jure Bajic
a2a6a3855b
Remove primary label from vertex (#529)
- Remove the primary label from vertex
- Pass vertex_validator instead of schema validator
- Fix vertex ctors
2022-09-07 13:36:10 +02:00
Jure Bajic
e9f0360fb3
Enable indices for storage v3 (#528)
- Enable indices in storage v3
- Add new test cases
- Change `CreateVertexAndValidate` to call `SetProperty` and `AddLabel`
2022-09-06 22:48:25 +02:00
Kostas Kyrimis
a2b04db23c Rename Middleware to ShardRequestManager 2022-09-06 13:12:48 +03:00
Kostas Kyrimis
5fa4e8c33a Merge project-pineapples 2022-09-06 13:05:48 +03:00
Kostas Kyrimis
57836f7c2b Added CreateVertices request test and polished 2022-09-05 22:28:15 +03:00
Kostas Kyrimis
c8bc4c7dbc Added Request ScanVertices on RequestClientManager and polished 2022-09-05 21:13:12 +03:00
János Benjamin Antal
fe4955447e
Merge branch 'project-pineapples' into E118-MG-lexicographically-ordered-storage 2022-09-05 14:59:01 +02:00
Tyler Neely
1631c20df2
In memory shard map (#505) 2022-09-05 15:50:54 +03:00
Kostas Kyrimis
e442bf435a Add abstract interface to Middleware and rename it. Cleaned up member function implementation and tests 2022-09-05 15:46:03 +03:00
Kostas Kyrimis
ba06d29a35 Added ScanAll prototype 2022-09-05 10:02:21 +03:00
Kostas Kyrimis
1a2f138e09 middleware prototype 2022-09-02 16:50:48 +03:00
János Benjamin Antal
947baedbe6 Merge remote-tracking branch 'origin/project-pineapples' into E118-MG-lexicographically-ordered-storage 2022-09-02 11:41:03 +02:00
Kostas Kyrimis
c6447eb48b
Add shard requests responses (#526) 2022-09-01 18:54:47 +03:00
Jure Bajic
7e84744d07
Split storage and shards (#519)
- Rename storage to shard
- Add primary label and range for shard
- Remove id_mapper functionality from shard
- Adapt tests
2022-09-01 09:10:40 +02:00
Tyler Neely
445cd2d206 Rename StorageClient in the tests to ShardClient to be consistent with ShardRsm 2022-08-31 14:42:57 +00:00
Tyler Neely
0fd448b300 Rename StorageRsm to ShardRsm 2022-08-31 14:40:53 +00:00
Tyler Neely
af4f3a5cb5 Add ShardMap::GetPropertyId 2022-08-31 14:24:20 +00:00
Tyler Neely
27459a9eb0 Add AllocatePropertyIds request/response pair for Coordinator + ShardMap 2022-08-31 14:22:40 +00:00
Tyler Neely
61b1fbfbc9 A number of clean-ups related to the coordinator and its tests 2022-08-30 15:14:28 +00:00
Tyler Neely
025df32d9e Small simplifications around compound keys 2022-08-30 14:19:50 +00:00
Tyler Neely
ab6585f19e Small clean-up of some TODOs and implementation of GetShardsForRange 2022-08-30 13:50:24 +00:00
Tyler Neely
c0d03888f4
Implement basic raft version (#498) 2022-08-30 15:07:34 +02:00
Tyler Neely
f96133e0b9 Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-30 13:00:35 +00:00
János Benjamin Antal
efb3c8d03d
Remove multi-threaded related logic and variables (#460)
* Remove logic that was necessary for optimal multi-threaded performance, such
  as accumulating deleted object in local containers and appending them to a
  global one, handling overlapping locking.
* Remove background GC thread.
* Remove mutexes, locks and atomics throughout storage.
2022-08-30 13:41:53 +02:00
Tyler Neely
2a395a18b0 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-30 08:38:14 +00:00
Tyler Neely
910c484c76 Simplify simulator code around reasoning about blocked participants 2022-08-30 08:22:54 +00:00
Tyler Neely
13787a0b17 Do not count clients waiting on promises as blocking when calculating quiescent counts in the simulator 2022-08-30 08:19:04 +00:00
Tyler Neely
46afb25415 use static cast for micros 2022-08-30 07:53:45 +00:00
Tyler Neely
62bb9a399a Revert incorrect code suggestion 2022-08-30 07:51:36 +00:00
Tyler Neely
ec95cd31ed
Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-30 09:12:54 +02:00
Tyler Neely
c0bbfc4f0e Address clang-tidy feedback 2022-08-29 15:42:19 +00:00
Tyler Neely
d50a6ab232 Properly use Follower's next_index for sending batches instead of the known committed_log_size 2022-08-29 14:13:40 +00:00
Tyler Neely
28fae9e6d0 Assert that terms in the log are monotonic 2022-08-29 14:01:30 +00:00
Tyler Neely
b7e54cbe44 Assert that terms in the log are monotonic 2022-08-29 14:00:29 +00:00
Tyler Neely
a410864855 Improve constness in several method bodies 2022-08-29 13:43:27 +00:00
Tyler Neely
325aae34d2 Optimize Cron calling order 2022-08-29 13:32:16 +00:00
Tyler Neely
8657909ef3 Simplify Handle(Leader, AppendResponse...) 2022-08-29 13:30:01 +00:00
Tyler Neely
726c42a3fc Use std::make_move_iterator for adding entries from AppendRequest into a Follower's Raft log 2022-08-29 13:24:34 +00:00
Jure Bajic
95dbc022c0
Integrate schema and keystore (#497)
- Integrate schema and keystore on vertex creation
- Add GC test for storage v3
- Add tests for accessors
- Fix all tests related to this except for query v2
- Fix labels not returning primary label
2022-08-29 14:38:25 +02:00
Tyler Neely
e67eefb1a3 Use a concept to lock-down certain Handle methods for Raft 2022-08-29 12:00:53 +00:00
Tyler Neely
a40403e3ce
Add local transport (#512)
* Create LocalTransport Io provider for sending messages to components on the same machine
* Move src/io/simulation/message_conversion.hpp to src/io/message_conversion.hpp for use in other Io providers
2022-08-29 13:49:51 +02:00
Tyler Neely
cb33a0a0c6 Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-29 11:35:24 +00:00
Tyler Neely
f6e41bd0f5 Make Raft index tracking less bug-prone by focusing on "log size" which gracefully initializes to 0 instead of 1-indexed log index tracking which does not cleanly initialize as intuitively 2022-08-29 09:18:25 +00:00
Tyler Neely
10f8af4681
Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-24 18:23:17 +02:00
Tyler Neely
87e9897c94 Use extracted constants for various timeout durations 2022-08-24 16:10:36 +00:00
Tyler Neely
34c3d94163 Fix and clarify comment 2022-08-24 16:01:19 +00:00
Tyler Neely
12c55d5d0b Remove several cases of unnecessary 'auto &&' 2022-08-24 15:58:46 +00:00
Tyler Neely
0d1616a5e0 Use Duration instead of std::chrono::microseconds which it aliases to 2022-08-24 15:55:10 +00:00
Tyler Neely
0be36a5805 Remove unnecessary aliases 2022-08-24 15:53:22 +00:00
Tyler Neely
8cea2937b8 Apply some feedback from clang-tidy 2022-08-24 15:00:52 +00:00
Tyler Neely
0b7f195f4b Remove unused include 2022-08-23 11:35:40 +00:00
Tyler Neely
409fe692d9 Check-in CoordinatorClient 2022-08-23 11:34:18 +00:00
Tyler Neely
7d1318b4d7 Clean up some rsm client and coordinator code 2022-08-19 15:17:33 +00:00
Tyler Neely
d42fa1fc87 Fix ShardMap::GetShardForKey 2022-08-18 15:14:04 +00:00
Tyler Neely
2a3a3338f0 Clean up coordinator a bit 2022-08-18 14:04:28 +00:00
Tyler Neely
2320f95dd1 Update the coordinator to include request for initializing a new shard map 2022-08-18 14:01:47 +00:00
Tyler Neely
3794693356 Make shard_rsm.hpp a bit more clear 2022-08-18 13:34:11 +00:00
Tyler Neely
d50b6c1abb Move rsm_client to src/io/rsm 2022-08-18 13:20:56 +00:00
Tyler Neely
d292162b00 Use new RsmClient in raft test 2022-08-17 16:40:18 +00:00
Tyler Neely
9c4556618a Rework RsmClient and update sharded_map commit 2022-08-17 16:35:35 +00:00
Tyler Neely
f0dc0d911d Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:51:09 +00:00
Tyler Neely
e545beaf78 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:47:13 +00:00
Tyler Neely
3c50b68954 Small comment fix 2022-08-16 16:47:13 +00:00
Tyler Neely
fad0c893f1 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:47:13 +00:00
Tyler Neely
866f1feeeb Added request for edge id batch request to coordinator rsm 2022-08-16 16:47:13 +00:00
gvolfing
16d1fa2c22 Gerneral clean-up 2022-08-16 16:47:13 +00:00
gvolfing
518e8df04c Resolve issues with the sharded_map test 2022-08-16 16:47:13 +00:00
gvolfing
9c453779f5 Add the coordinator part to the sharded_map test 2022-08-16 16:47:13 +00:00
gvolfing
5d66a4e828 Check against cas_succeeded 2022-08-16 16:47:13 +00:00
gvolfing
2553a8fdc3 Remove unfinished statement 2022-08-16 16:47:13 +00:00
gvolfing
9aeca7a4b3 Add test for sharded_map 2022-08-16 16:47:13 +00:00
gvolfing
1b2c8f6b29 Move RsmClient into a separate folder and header 2022-08-16 16:47:13 +00:00
gvolfing
a74a556fcc Remove unnecessary storage pool, general cleanup 2022-08-16 16:47:13 +00:00
gvolfing
66e791d042 Make Coordinator apply splitting on request. 2022-08-16 16:47:13 +00:00
gvolfing
797c76cdfd Add logic to split shards 2022-08-16 16:47:13 +00:00
gvolfing
7af917e408 Add abstraction for RsmClient 2022-08-16 16:47:13 +00:00
gvolfing
c7282e8935 Temporarly remove the stanby server pool 2022-08-16 16:47:13 +00:00
gvolfing
4ed5801588 Change min and max key related shard logic 2022-08-16 16:47:13 +00:00
gvolfing
5963c83a60 Update Coordinator 2022-08-16 16:47:13 +00:00
gvolfing
edf1293274 Add GetShardMap read-only request to coordinator 2022-08-16 16:47:13 +00:00
gvolfing
dd46cc407f Return an error and the latest known ShardMap version if the requested key is not possibly stored in the given shard 2022-08-16 16:47:13 +00:00
Tyler Neely
92d69e080c Add todos to coordinator.hpp 2022-08-16 16:47:13 +00:00
Tyler Neely
629fd231b3 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:47:09 +00:00
Tyler Neely
95e90e6c2e Check-in in-progress shard test 2022-08-16 16:42:28 +00:00
Tyler Neely
502a9b4823 Check-in coordinator_rsm.hpp 2022-08-16 16:42:28 +00:00
Tyler Neely
c62b0eff93 Add Simulator::RegisterNew helper method 2022-08-16 16:42:28 +00:00
Tyler Neely
6b9311e0b8 Bump shard_rsm to use new method names 2022-08-16 16:42:28 +00:00
Tyler Neely
e3dd404865 Add new test, start to fill out coordinator RSM 2022-08-16 16:42:28 +00:00
Tyler Neely
523e2b9186 Merge branch 'T0912-MG-in-memory-shard-map' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:42:28 +00:00
Tyler Neely
79539d13c9 Use Read/Apply instead of read/apply in Rsm concept 2022-08-16 16:42:28 +00:00
Tyler Neely
6169eb221b
Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:13:41 +02:00
Tyler Neely
342611691a
Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:13:30 +02:00
Tyler Neely
c256dce601
Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:13:03 +02:00
Tyler Neely
2e9cf8f37d
Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:12:13 +02:00
Tyler Neely
47186cab18
Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:11:56 +02:00
Tyler Neely
dcc5ec920a
Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:11:41 +02:00
Tyler Neely
c7d96ed5c5
Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:11:11 +02:00
Tyler Neely
7ad9b62968
Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:10:37 +02:00
Tyler Neely
bb44aaa188
Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:09:51 +02:00
Tyler Neely
9961659103 Use spdlog instead of cout 2022-08-16 14:52:03 +00:00
Tyler Neely
6f906c0488 Use spdlog instead of cout 2022-08-16 14:44:17 +00:00
Tyler Neely
d161163fb4 Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-15 13:49:50 +00:00
Tyler Neely
1423da5f51 Improve comments around majority index selection 2022-08-15 11:57:19 +00:00
Tyler Neely
5e4733ac98 Add extra safety checks to commit_index management 2022-08-12 13:20:11 +00:00
Tyler Neely
14c9e68456
Transport prototype (#466) 2022-08-12 08:24:32 +02:00
Tyler Neely
cbdfd54e5f Properly apply state updates in Follower Handle for AppendRequest 2022-08-11 12:30:26 +00:00
Tyler Neely
69a2d73172 Fix bug-prone leader application of write requests to the replicated state 2022-08-11 12:17:33 +00:00
Tyler Neely
5b69f435b6 Small comment cleanup 2022-08-11 08:39:11 +00:00
Tyler Neely
f098d6b331 Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-10 10:48:17 +00:00
Tyler Neely
f6edce8f0b Apply Kostas's feedback to transport interface 2022-08-10 09:34:02 +00:00
Tyler Neely
1225230c3a Apply feedback from cpplint 2022-08-09 15:52:24 +00:00
Tyler Neely
5f225a623e align cpp w/ hpp naming 2022-08-09 15:21:52 +00:00
Tyler Neely
d15105455a Clean up comments and TODOs 2022-08-09 14:07:42 +00:00
Tyler Neely
36c133ce4e Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-08 16:29:48 +00:00
Tyler Neely
91d48f05f3
Merge branch 'project-pineapples' into T0879-MG-transport-prototype 2022-08-08 13:34:30 +02:00
Tyler Neely
7bac0d1c61 Use default equality operator for Address 2022-08-08 11:32:42 +00:00
Tyler Neely
a4b1d5efb4
Update src/io/future.hpp
Co-authored-by: Jure Bajic <jure.bajic@memgraph.com>
2022-08-08 13:27:06 +02:00
jbajic
68b26275a3 Merge branch 'project-pineapples' into E118-MG-lexicographically-ordered-storage 2022-08-08 11:34:46 +02:00
Tyler Neely
0a43afdec1 Update raft implementation to use std::chrono like upstream 2022-08-04 13:28:52 +00:00
Tyler Neely
618a3d96b3 Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-04 12:07:54 +00:00
jbajic
5012824e05 Address review comments 2022-08-04 11:45:16 +02:00
gvolfing
ee16954641 Add minimum and maximum key for shard rsm prototype 2022-08-04 11:20:19 +02:00
gvolfing
0aab854e34 Add shard rsm prototype 2022-08-04 10:47:00 +02:00
Jure Bajic
a12a1ea358
Move schema to storage v3 and query v2
* Move schema to storage v3

* Remove schema from v2

* Move schema to query v2

* Remove schema from query v1

* Make glue v2

* Move schema related tests to newer versions of query and storage

* Fix typo in CMake

* Fix interpreter test

* Fix clang tidy errors

* Change temp dir name
2022-08-04 09:50:02 +02:00
Tyler Neely
343648f564 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-04 07:49:09 +00:00
gvolfing
726fabd387 Constrain Raft class template with concept 2022-08-04 09:09:18 +02:00
János Benjamin Antal
2891041468
Make storage use KeyStore (#455) 2022-08-03 18:10:58 +02:00
Tyler Neely
d1c5aead61 Implement the rest of the KV-on-Raft RSM test 2022-08-03 15:07:08 +00:00
Tyler Neely
aebac2c519 Hoist read and write requests on the RSM into wrapper structs for futureproofing 2022-08-03 14:21:37 +00:00
Tyler Neely
74b354979c Check in incremental progress on fixing compilation error 2022-08-03 13:42:08 +00:00
gvolfing
b5cff5999b Unfinished rsm changes 2022-08-03 14:39:09 +02:00
Tyler Neely
b2a8063a96 Make more temporaries const 2022-08-02 14:33:05 +00:00
Tyler Neely
54369958d1 Extract BlockedServers functionality from MaybeTickSimulator and IncrementServerCountAndWaitForQuiescentState 2022-08-02 14:30:12 +00:00
Tyler Neely
4f06eb0f2f Address rule-of-five for abstract inhertance in OpaquePromise 2022-08-02 14:24:59 +00:00
Tyler Neely
ad0a8c4942 Use simple virtual inheritance instead of bespoke vtable for OpaquePromise 2022-08-02 13:36:40 +00:00
Tyler Neely
997fdf5a16 Check-in skeleton for RSM logic on top of Raft 2022-08-02 10:18:38 +00:00
Tyler Neely
8be88deee6 Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-02 08:48:20 +00:00
Tyler Neely
902a46d14f Make dynamic message Take messages rvalue methods 2022-08-02 07:12:05 +00:00
Tyler Neely
3b44ef70b6 Denest operator< for clarity 2022-08-02 07:06:43 +00:00
Tyler Neely
4d8f9ea821 Remove unnecessary comment 2022-08-02 07:03:12 +00:00
Tyler Neely
e935a9a7b1 Make parenthesization less confusing 2022-08-02 07:02:17 +00:00
Tyler Neely
dd93b594bc Remove dead code 2022-08-02 07:01:25 +00:00
János Benjamin Antal
cc5ee6a496 Merge remote-tracking branch 'origin/project-pineapples' into E118-MG-lexicographically-ordered-storage 2022-08-02 08:19:43 +02:00
Tyler Neely
997d25d536 Remove buggy usage of std::forward 2022-08-01 15:53:37 +00:00
Tyler Neely
ace5f2b639 Complete clang-tidy cleanup 2022-08-01 15:44:34 +00:00
Tyler Neely
9576eea051 Address some feedback from clang-tidy 2022-08-01 15:37:16 +00:00
Tyler Neely
5bb3361a2d Use std::chrono::microseconds explicitly for Duration. Fix compiler warning related to timeouts 2022-08-01 14:32:07 +00:00
gvolfing
a84f5f6115 Undo the structured binding changes 2022-08-01 16:13:29 +02:00
Tyler Neely
cacb0dac80 Avoid unnamed namespace in future.hpp 2022-08-01 14:12:07 +00:00
gvolfing
918fa7212e Further improve const-correctness and replace dispensable structured bindings with exact return values 2022-08-01 15:30:44 +02:00
gvolfing
e905591372 Add references to the raft paper 2022-08-01 15:17:29 +02:00
Tyler Neely
649b5437b0 Improve docs around Io interface 2022-08-01 12:58:55 +00:00
Tyler Neely
102d997288
Update src/io/simulator/simulator_handle.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-01 14:52:06 +02:00
Tyler Neely
a3f3e05fc2 Include replier address in operator< for PromiseKey 2022-08-01 12:50:48 +00:00
Tyler Neely
4f4eb9ea13
Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-01 14:47:13 +02:00
Tyler Neely
ca638db509 Remove extra blank line 2022-08-01 12:45:45 +00:00
Tyler Neely
7d33bb1937 Update Future unit test to use gtest 2022-08-01 12:42:37 +00:00
Tyler Neely
b8487da392 Move future test to unit tests 2022-08-01 11:58:54 +00:00
Tyler Neely
dbd744470b Add future benchmark 2022-08-01 11:48:21 +00:00
Tyler Neely
b2b11f3a30 Run simulation tests in CI 2022-08-01 11:24:21 +00:00
Tyler Neely
eb1b6c3ac8 Make SimulatorHandle::TimeoutPromisesPastDeadline private 2022-08-01 11:10:00 +00:00
Tyler Neely
0c2cbb5461 Make a couple more things const 2022-08-01 11:08:17 +00:00
Tyler Neely
69ea79a75e Make a couple more things const 2022-08-01 11:07:39 +00:00
Tyler Neely
c0d6cec9ab Split simulator_handle.cpp into a source and header file 2022-08-01 11:06:08 +00:00
Tyler Neely
9b915be1aa Make some methods const. Remove outdated TODO 2022-08-01 10:30:37 +00:00
Tyler Neely
b127f6f345 Use spdlog and create a formatter for Address 2022-08-01 10:23:25 +00:00
jbajic
f57f30c8cf Merge branch 'project-pineapples' into E112-MG-implement-partial-schema 2022-08-01 10:46:11 +02:00
Tyler Neely
66ef5b2072 Make certain temporaries const 2022-08-01 08:26:09 +00:00
Tyler Neely
f877f8e1d3 Add some requests and additional metadata around coordinator operations 2022-07-29 14:40:08 +00:00
gvolfing
0ea9878fd1 CTAD 2022-07-29 14:27:55 +02:00
gvolfing
af94270dc6 Remove redundant else clauses when returning from every branch 2022-07-29 14:22:57 +02:00
gvolfing
8aac3ae7ea Make unchanged variables const 2022-07-29 14:08:37 +02:00
Tyler Neely
507018a630 Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-07-29 11:50:21 +00:00
Tyler Neely
e0bbf4766a Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0879-MG-transport-prototype 2022-07-29 11:47:50 +00:00
Tyler Neely
1a48e0ffa8 Merge github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-07-29 11:46:54 +00:00
Jure Bajic
462daf3a2b
Enforce schema on vertex creation
- Separating schema definition from schema validation
- Updating vertex_accessor and db_accessors with necessary methods
- Adding a primary label to Vertex
- Adding schema tests
- Updating existing tests for storage v3, and deprecating old:
  - interpreter => interpreter_v2
  - query_plan_accumulate_aggregate => storage_v3_query_plan_accumulate_aggregate
  - query_plan_create_set_remove_delete => storage_v3_query_plan_create_set_remove_delete
  - query_plan_bag_semantics => storage_v3_query_plan_bag_semantics
  - query_plan_edge_cases => storage_v3_query_plan_edge_cases
  - query_plan_v2_create_set_remove_delete => storage_v3_query_plan_v2_create_set_remove_delete
  - query_plan_match_filter_return => storage_v3_query_plan_match_filter_return
2022-07-29 13:38:17 +02:00
Tyler Neely
6a808ce1fc Delete empty line 2022-07-28 16:27:57 +00:00
Tyler Neely
a89a2d9caa
Update src/io/simulator/simulator_transport.hpp
Co-authored-by: Jure Bajic <jure.bajic@memgraph.com>
2022-07-28 18:26:36 +02:00
Tyler Neely
c4f1764fee Rely on default initialization of random number generators 2022-07-28 16:24:45 +00:00
Tyler Neely
48a445f2ed Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0879-MG-transport-prototype 2022-07-28 16:24:34 +00:00
Tyler Neely
406da4b25c Use uint8_t instead of int for SimulatorConfig::drop_percent 2022-07-28 16:23:10 +00:00
Tyler Neely
8fde05444d
Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-28 18:13:50 +02:00
Tyler Neely
6bdfb43ad0 Use rvalue reference requirement for Future::Wait 2022-07-28 16:07:20 +00:00
Tyler Neely
9056e2c97a Make Address's == and < operators consider uuid 2022-07-28 16:06:48 +00:00
Marko Budiselic
2009fefc8a Merge branch 'master' into project-pineapples 2022-07-28 15:36:51 +02:00
Marko Budiselić
eb3f96d1f6
Bring changes from master to project-pineapples (#477)
* Fix aggregation functions on `null` and group-by inputs (#448)
* Upgrade Antrl to 4.10.1 and remove antlr_lock (#441)
* Update clang-tidy job (#476)
* Add parser stress test (#463)

NOTE: Doing this to have buildable comments on the project-pineapples branch

Co-authored-by: gvolfing <107616712+gvolfing@users.noreply.github.com>
Co-authored-by: Jure Bajic <jure.bajic@memgraph.com>
2022-07-28 15:36:17 +02:00
Tyler Neely
51371398ce
Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 17:27:11 +02:00
Tyler Neely
b38dc28e01
Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 17:26:00 +02:00
Tyler Neely
6cc550719b
Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 17:25:25 +02:00
Tyler Neely
ef70b858e2 Centralize Future's Shared state taking logic. Fix a race condition in quiescence tracking 2022-07-27 15:24:33 +00:00
Tyler Neely
509c12956c
Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 17:09:32 +02:00
Tyler Neely
1527509e36 Use std::chrono instead of raw uint64_t 2022-07-27 15:07:01 +00:00
Tyler Neely
0cfb68bb89
Update src/io/simulator/simulator_handle.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 16:56:14 +02:00
Tyler Neely
e2968c2e21 Use std::chrono instead of raw uint64_t 2022-07-27 14:55:39 +00:00
Tyler Neely
4ee4612a9c Remove removed raft.cpp from CMakeLists.txt for the simulator tests 2022-07-27 14:06:16 +00:00
Tyler Neely
dc38296575 avoid unnecessary std::optional around std::function which can be nullable 2022-07-27 14:05:57 +00:00
Tyler Neely
06962a3ec4 Check shard map version 2022-07-26 15:35:37 +00:00
Tyler Neely
c0b0b08d12 Check-in deregistration and shard split things 2022-07-26 15:32:22 +00:00
Tyler Neely
c100a86644 Check-in changes from hangout 2022-07-26 15:23:55 +00:00
Tyler Neely
3ec1ff9ee4 Commit small changes from meeting 2022-07-26 11:49:08 +00:00
Tyler Neely
98206caf85 Separate split functionality from ShardMap 2022-07-25 10:38:50 +00:00
Tyler Neely
962767ea1c Initial check-in of shard map ideas 2022-07-25 10:34:27 +00:00
Tyler Neely
2a199c9484 Merge branch 'master' of github.com:memgraph/memgraph into T0879-MG-transport-prototype 2022-07-22 12:14:18 +00:00
Tyler Neely
a85e9fcdd4 Remove raft-related code from transport prototype branch 2022-07-22 12:11:00 +00:00
jbajic
264b233053 Merge branch 'project-pineapples' into E112-MG-implement-partial-schema 2022-07-22 11:48:45 +02:00
Kostas
72b4337864 Added ScanVertices request/response example 2022-07-21 18:37:06 +03:00
János Benjamin Antal
ed71332773 Add dummy storage engine to test 2022-07-21 16:23:27 +02:00
Tyler Neely
fd3d70d847 Move raft-related code to replicated state machine (rsm) namespace 2022-07-21 12:41:02 +00:00
Tyler Neely
f8e5032011 Restructure files into namespaces and subdirectories 2022-07-21 12:10:23 +00:00
Tyler Neely
5b59d890c0 Remove awkward Reply sugar for now 2022-07-21 11:53:51 +00:00
Tyler Neely
0351db2461 Move future.hpp to src/io 2022-07-21 11:45:05 +00:00
Tyler Neely
c379475e12 Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0879-MG-transport-prototype 2022-07-21 11:01:04 +00:00
Tyler Neely
9c5d19bc19 Start to adapt the transport-related code to use namespaces 2022-07-21 11:00:11 +00:00
Tyler Neely
4d85a7e605 Check-in simulator_transport.hpp 2022-07-21 10:49:33 +00:00
Tyler Neely
f6c2202772 Use anonymous namespace to avoid friend declarations in future.hpp 2022-07-21 10:49:19 +00:00
Tyler Neely
498ae97ae9 Clean up log messages 2022-07-21 10:26:08 +00:00
Tyler Neely
fbd015d3c6 Continue codebase clean-up in preparation for merge 2022-07-21 10:24:13 +00:00
Tyler Neely
1ef11a36f4 Apply feedback from cpplint 2022-07-21 09:33:06 +00:00
Tyler Neely
9ae1671e4f Fix bug with MgFuture where its safety check was not correctly initialized 2022-07-20 18:17:06 +00:00
Tyler Neely
b4afe45de5 Handle Request timeouts properly in the simulator 2022-07-20 17:18:07 +00:00
Tyler Neely
581925e660 Fix a few bugs, expose stats 2022-07-20 17:07:18 +00:00
Tyler Neely
afef6dc11b A large batch of bug fixes and clean-ups 2022-07-20 13:18:00 +00:00
Tyler Neely
689336e765 Fix several bugs, clean-ups and tuning 2022-07-19 14:57:01 +00:00
Jure Bajic
2ceaf59767
Create query engine v2 (#444)
Create version v2 of the query engine.
Adjust CMake and lisp files
Connect query engine v2 with storage engine v3
2022-07-19 12:28:19 +02:00
Marko Budiselic
5dd0ddc352 Killing bugs 2022-07-18 11:36:56 +02:00
János Benjamin Antal
c0bee760bf git 2022-07-18 08:21:04 +02:00
Marko Budiselic
1480d975a7 Fix the basic_request example, something sill broken 2022-07-17 17:22:39 +02:00
Tyler Neely
9dc37a87f7 Enable more aggressive message scrambling to get a race condition to jump out about 0.2% of the time 2022-07-15 16:46:54 +00:00
Tyler Neely
46a2879ece Fix a number of bugs in raft, and it can now go for around 1400 simulations before hitting a race condition 2022-07-15 16:44:42 +00:00
Tyler Neely
b2142e8d38 Remove unnecessary log message 2022-07-15 16:20:05 +00:00
Tyler Neely
0dc69c180d Start cranking up the testing intensity and fix a variety of correctness and code quality issues 2022-07-15 16:19:00 +00:00
Tyler Neely
ad1d8637e5 Fix race condition in simulator. Make leader election function more reliably in raft code 2022-07-15 09:07:39 +00:00
Tyler Neely
4876b7cd8c Start removing load-bearing sleep statements to get race conditions to jump out in the simulator 2022-07-15 05:43:40 +00:00
Tyler Neely
a6133dab49 Start to tune various timeouts in the raft implementation 2022-07-15 05:38:04 +00:00
Tyler Neely
3466c15f76 Make raft work in the happy path 2022-07-14 19:57:52 +00:00
Tyler Neely
63beeb8771 Send log state along with AppendRequest messages from Leader 2022-07-14 19:06:56 +00:00
Tyler Neely
d14f7705b1 Get most of the raft data path up and running 2022-07-14 18:45:24 +00:00
Tyler Neely
5e98971bb2 Continue filling out AppendEntries state transition logic 2022-07-14 13:06:54 +00:00
Tyler Neely
dc78adde40 Move empty in-flight early exit of simulator tick to before time advancement 2022-07-14 13:06:22 +00:00
Tyler Neely
80970a97f0 Get basic leader election to work and clean some things up 2022-07-14 12:18:09 +00:00
Tyler Neely
57533f2746 A large number of scattered cleanups. Don't use transport Request in state machine code 2022-07-14 10:50:49 +00:00
Tyler Neely
1a12a80af0 Continue to implement raft state machine transitions 2022-07-14 07:09:50 +00:00
Tyler Neely
30c97e658d Implement some more of the raft state machine transitions 2022-07-13 22:31:51 +00:00
Tyler Neely
dd9862d32c Add support for scrambling messages to the simulator 2022-07-13 19:59:23 +00:00
Tyler Neely
dc6548c996 Add support to the simulator to drop messages randomly 2022-07-13 19:51:04 +00:00
Tyler Neely
881e914b92 Add random number generator support to the IO interface 2022-07-13 13:37:38 +00:00
Tyler Neely
48ee40ce87 Start to fill out more raft logic now that role-specialized Handle and Cron are in place 2022-07-13 06:50:16 +00:00
Tyler Neely
d6742f643c Significant simplification of event-handling logic by using std::visit while matching on both the message type and node role 2022-07-12 19:04:32 +00:00
Tyler Neely
d8f09b59b3 Add MgFuture::Cancel to signal that a future is intentionally dropped 2022-07-12 19:03:12 +00:00
Tyler Neely
2e5d8b7e8c Check-in in-progress work for raft test on top of simulator 2022-07-11 18:01:13 +00:00
Tyler Neely
88eee66258 Small cleanup of basic_request test 2022-07-11 18:00:30 +00:00
Tyler Neely
d4cb259979 Fix issue with recursive parameter pack std::variant conversion of std::any. Improve documentation of the complex functionality 2022-07-11 17:59:52 +00:00
Tyler Neely
1a87dd2497 Add two methods to MgFuture: IsReady and TryGet 2022-07-11 17:59:08 +00:00
Jure Bajic
3f4f66b57f
Create schema DDL expressions
* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Add schema operations in storage

* Add create and show schema queries

* Add privileges for schema

* Add missing keywords into lexer

* Add drop schema query

* Add schema visitors

* Update metadata

* Add PrepareSchemaQuery function

* Implement show schemas

* Add show schema query

* Fix schema visitor

* Add common schema type

* Fix grammar

* Temporary create ddl logic

* Fix naming for schemaproperty type to schema type

* Rename schemaproperty to schemapropertytype

* Enable Create schema ddl

* Override visitPropertyType

* Add initial schema implementation

* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Apply suggestions from code review

Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

* Add verification on creation and deletion

* Rename DeleteSchema to DropSchema

* Remove list and map from lexer

* Fix grammar with schemaTypeMap

* Add privilege and cypher visitor tests

* Catch repeating type name in schema definition

* Fix conflicting keywords

* Add notifications

* Drop float support

* Finish interpreter tests

* Fix tests

* Fix clang tidy errors

* Fix GetSchema

* Replace for with transfrom

* Add cloning og schema_property_map

* Address review comments

* Rename SchemaPropertyType to SchemaType

* Remove inline

* Assert of schema properties

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>
2022-07-11 09:20:15 +02:00
Tyler Neely
15d637729e Clean-up basic_request example 2022-07-08 14:10:33 +00:00
Tyler Neely
5ee95a2e70 Add ShutDown method to Simulator and update basic_request test to utilize it 2022-07-08 14:05:02 +00:00
Tyler Neely
597a5d191c Check-in SimulatorConfig 2022-07-08 11:49:29 +00:00
Tyler Neely
eee2a7e019 Fix memory leak caught by LSAN 2022-07-08 11:04:44 +00:00
Jure Bajic
2998f92595 Add initial schema implementation
* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Apply suggestions from code review

Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>
2022-07-08 10:33:43 +02:00
Tyler Neely
25b0a445b9 Properly fill response promises in the simulator 2022-07-08 08:21:37 +00:00
Tyler Neely
20a6dae047 Get the message receive path up and running. Large number of small refactors 2022-07-07 20:58:16 +00:00
Tyler Neely
04dbedc3af Add functionality for converting from std::any to a std::variant. Improve type names. Continue simulator implementation 2022-07-07 11:27:02 +00:00
Tyler Neely
a0058bc10a Add OpaquePromise type with correct dtor. Make a few structs comparable for storage in maps. Fix initialization of Shared 2022-07-06 15:19:16 +00:00
Tyler Neely
f601e83f6f Continue structuring the simulator and filling in its implementation 2022-07-05 21:11:39 +00:00
Tyler Neely
e1aab7065f Use TSAN on futures test. Continue implementation of simulator 2022-07-05 18:02:01 +00:00
Tyler Neely
28516763b9 Fix issue with movement/consumption tracking in MgFuture and MgPromise 2022-07-05 17:36:47 +00:00
Tyler Neely
29e8d8e72c Fix cmake config for new transport 2022-07-05 16:04:03 +00:00
Tyler Neely
6debc9e7d8 Revamp MgFuture, continue threading logic through Simulator 2022-07-05 15:45:59 +00:00
János Benjamin Antal
1bdc32ba5d
Copy storage v2 to create storage v3 (#416)
* Copy storage v2 to v3

* Integrate v3 to cmake

* Fix clang-tidy warnings

* Add dummy unit test for storage-v3 to trigger build for code analysis builds
2022-07-05 08:20:59 +02:00
Tyler Neely
eb4ca543ea Continue to implement the simulated transport 2022-07-04 21:42:10 +00:00
Tyler Neely
38ca430713 Small refactor of cmake for simulator test 2022-07-04 20:44:51 +00:00
Tyler Neely
24128e0bca Check-in new test 2022-07-04 19:42:23 +00:00
Tyler Neely
73719b2120 Continue to bolt-down MgFuture impl 2022-07-04 18:54:09 +00:00
Tyler Neely
20839b0ae0 Add simple test for MgFuture 2022-07-04 15:00:32 +00:00
Tyler Neely
6cec9acbb9 Add basic test skeleton 2022-07-04 11:56:41 +00:00
Tyler Neely
cb70431301 Reset optional after moving its value out in Future::Wait 2022-07-04 11:55:18 +00:00
Tyler Neely
0ef1f7eb5d Apply Rule-Of-Five and uniform CamelCase function names 2022-07-01 08:45:19 +00:00
Tyler Neely
4140f3e05e [SQUASH ME] Initial check-in for getting feedback 2022-06-29 15:52:37 +00:00
János Benjamin Antal
21870a0e7e Merge branch 'master' into project-pineapples 2022-06-23 14:49:14 +02:00
258 changed files with 86740 additions and 41 deletions

View File

@ -6,6 +6,7 @@ Checks: '*,
-altera-unroll-loops,
-android-*,
-cert-err58-cpp,
-cert-str34-c,
-cppcoreguidelines-avoid-c-arrays,
-cppcoreguidelines-avoid-goto,
-cppcoreguidelines-avoid-magic-numbers,
@ -49,6 +50,7 @@ Checks: '*,
-misc-non-private-member-variables-in-classes,
-modernize-avoid-c-arrays,
-modernize-concat-nested-namespaces,
-modernize-loop-convert,
-modernize-pass-by-value,
-modernize-use-equals-default,
-modernize-use-nodiscard,

View File

@ -130,7 +130,7 @@ jobs:
source /opt/toolchain-v4/activate
# Restrict clang-tidy results only to the modified parts
git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -extra-arg="-DMG_CLANG_TIDY_CHECK" -path build | tee ./build/clang_tidy_output.txt
# Fail if any warning is reported
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
@ -171,7 +171,7 @@ jobs:
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
ctest -E "(memgraph__unit|memgraph__benchmark|memgraph__simulation)" --output-on-failure
- name: Run drivers tests
run: |
@ -262,6 +262,15 @@ jobs:
cd build
ctest -R memgraph__unit --output-on-failure -j$THREADS
- name: Run simulation tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
ctest -R memgraph__simulation --output-on-failure -j$THREADS
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.

View File

@ -39,7 +39,7 @@ jobs:
source /opt/toolchain-v4/activate
# The results are also written to standard output in order to retain them in the logs
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v4/bin/clang-tidy "$PWD/src/*" |
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -extra-arg="-DMG_CLANG_TIDY_CHECK" -clang-tidy-binary=/opt/toolchain-v4/bin/clang-tidy "$PWD/src/*" |
tee ./build/full_clang_tidy_output.txt
- name: Summarize clang-tidy results

12
.gitignore vendored
View File

@ -23,6 +23,8 @@ cmake-build-*
cmake/DownloadProject/
dist/
src/query/frontend/opencypher/generated/
src/query/v2/frontend/opencypher/generated/
src/parser/opencypher/generated
tags
ve/
ve3/
@ -50,15 +52,25 @@ src/distributed/pull_produce_rpc_messages.hpp
src/distributed/storage_gc_rpc_messages.hpp
src/distributed/token_sharing_rpc_messages.hpp
src/distributed/updates_rpc_messages.hpp
src/query/v2/frontend/ast/ast.hpp
src/query/frontend/ast/ast.hpp
src/storage/v3/bindings/ast/ast.hpp
src/query/distributed/frontend/ast/ast_serialization.hpp
src/query/v2/distributed/frontend/ast/ast_serialization.hpp
src/durability/distributed/state_delta.hpp
src/durability/single_node/state_delta.hpp
src/durability/single_node_ha/state_delta.hpp
src/query/frontend/semantic/symbol.hpp
src/query/v2/frontend/semantic/symbol.hpp
src/expr/semantic/symbol.hpp
src/query/distributed/frontend/semantic/symbol_serialization.hpp
src/query/v2/distributed/frontend/semantic/symbol_serialization.hpp
src/query/distributed/plan/ops.hpp
src/query/v2/distributed/plan/ops.hpp
src/query/plan/operator.hpp
src/query/v2/plan/operator.hpp
src/parser/opencypher/generated
src/expr/semantic/symbol.hpp
src/raft/log_entry.hpp
src/raft/raft_rpc_messages.hpp
src/raft/snapshot_metadata.hpp

View File

@ -5,16 +5,22 @@ add_subdirectory(lisp)
add_subdirectory(utils)
add_subdirectory(requests)
add_subdirectory(io)
add_subdirectory(io/simulator)
add_subdirectory(kvstore)
add_subdirectory(telemetry)
add_subdirectory(communication)
add_subdirectory(memory)
add_subdirectory(storage/v2)
add_subdirectory(storage/v3)
add_subdirectory(integrations)
add_subdirectory(query)
add_subdirectory(query/v2)
add_subdirectory(slk)
add_subdirectory(rpc)
add_subdirectory(auth)
add_subdirectory(parser)
add_subdirectory(expr)
add_subdirectory(coordinator)
if (MG_ENTERPRISE)
add_subdirectory(audit)

View File

@ -37,7 +37,7 @@ const std::vector<Permission> kPermissionsAll = {
Permission::CONSTRAINT, Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY, Permission::TRIGGER,
Permission::CONFIG, Permission::STREAM, Permission::MODULE_READ, Permission::MODULE_WRITE,
Permission::WEBSOCKET};
Permission::WEBSOCKET, Permission::SCHEMA};
} // namespace
std::string PermissionToString(Permission permission) {
@ -84,6 +84,8 @@ std::string PermissionToString(Permission permission) {
return "MODULE_WRITE";
case Permission::WEBSOCKET:
return "WEBSOCKET";
case Permission::SCHEMA:
return "SCHEMA";
}
}

View File

@ -38,7 +38,8 @@ enum class Permission : uint64_t {
STREAM = 1U << 17U,
MODULE_READ = 1U << 18U,
MODULE_WRITE = 1U << 19U,
WEBSOCKET = 1U << 20U
WEBSOCKET = 1U << 20U,
SCHEMA = 1U << 21U
};
// clang-format on

19
src/common/types.hpp Normal file
View File

@ -0,0 +1,19 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cstdint>
namespace memgraph::common {
enum class SchemaType : uint8_t { BOOL, INT, STRING, DATE, LOCALTIME, LOCALDATETIME, DURATION };
} // namespace memgraph::common

View File

@ -0,0 +1,9 @@
set(coordinator_src_files
coordinator.cpp
shard_map.cpp)
find_package(fmt REQUIRED)
find_package(Threads REQUIRED)
add_library(mg-coordinator STATIC ${coordinator_src_files})
target_link_libraries(mg-coordinator stdc++fs Threads::Threads fmt::fmt mg-utils mg-storage-v3)

View File

@ -0,0 +1,129 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <coordinator/coordinator.hpp>
namespace memgraph::coordinator {
CoordinatorWriteResponses Coordinator::ApplyWrite(HeartbeatRequest &&heartbeat_request) {
spdlog::info("Coordinator handling HeartbeatRequest");
// add this storage engine to any under-replicated shards that it is not already a part of
auto initializing_rsms_for_shard_manager =
shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
return HeartbeatResponse{
.shards_to_initialize = initializing_rsms_for_shard_manager,
};
}
CoordinatorWriteResponses Coordinator::ApplyWrite(HlcRequest &&hlc_request) {
HlcResponse res{};
auto hlc_shard_map = shard_map_.GetHlc();
MG_ASSERT(!(hlc_request.last_shard_map_version.logical_id > hlc_shard_map.logical_id));
res.new_hlc = Hlc{
.logical_id = ++highest_allocated_timestamp_,
// TODO(tyler) probably pass some more context to the Coordinator here
// so that we can use our wall clock and enforce monotonicity.
// .coordinator_wall_clock = io_.Now(),
};
// Allways return fresher shard_map for now.
res.fresher_shard_map = std::make_optional(shard_map_);
return res;
}
CoordinatorWriteResponses Coordinator::ApplyWrite(AllocateEdgeIdBatchRequest &&ahr) {
AllocateEdgeIdBatchResponse res{};
uint64_t low = highest_allocated_edge_id_;
highest_allocated_edge_id_ += ahr.batch_size;
uint64_t high = highest_allocated_edge_id_;
res.low = low;
res.high = high;
return res;
}
/// This splits the shard immediately beneath the provided
/// split key, keeping the assigned peers identical for now,
/// but letting them be gradually migrated over time.
CoordinatorWriteResponses Coordinator::ApplyWrite(SplitShardRequest &&split_shard_request) {
SplitShardResponse res{};
if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
res.success = false;
} else {
res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
split_shard_request.split_key);
}
return res;
}
/// This adds the provided storage engine to the standby storage engine pool,
/// which can be used to rebalance storage over time.
CoordinatorWriteResponses Coordinator::ApplyWrite(
RegisterStorageEngineRequest && /* register_storage_engine_request */) {
RegisterStorageEngineResponse res{};
// TODO
return res;
}
/// This begins the process of draining the provided storage engine from all raft
/// clusters that it might be participating in.
CoordinatorWriteResponses Coordinator::ApplyWrite(
DeregisterStorageEngineRequest && /* register_storage_engine_request */) {
DeregisterStorageEngineResponse res{};
// TODO
return res;
}
CoordinatorWriteResponses Coordinator::ApplyWrite(InitializeLabelRequest &&initialize_label_request) {
InitializeLabelResponse res{};
std::optional<LabelId> new_label_id = shard_map_.InitializeNewLabel(
initialize_label_request.label_name, initialize_label_request.schema, initialize_label_request.replication_factor,
initialize_label_request.last_shard_map_version);
if (new_label_id) {
res.new_label_id = new_label_id.value();
res.fresher_shard_map = std::nullopt;
res.success = true;
} else {
res.fresher_shard_map = shard_map_;
res.success = false;
}
return res;
}
CoordinatorWriteResponses Coordinator::ApplyWrite(AllocatePropertyIdsRequest &&allocate_property_ids_request) {
AllocatePropertyIdsResponse res{};
auto property_ids = shard_map_.AllocatePropertyIds(allocate_property_ids_request.property_names);
res.property_ids = property_ids;
return res;
}
} // namespace memgraph::coordinator

View File

@ -0,0 +1,196 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <optional>
#include <set>
#include <string>
#include <unordered_set>
#include <variant>
#include <vector>
#include <boost/uuid/uuid.hpp>
#include <coordinator/hybrid_logical_clock.hpp>
#include <coordinator/shard_map.hpp>
#include <io/simulator/simulator.hpp>
#include <io/time.hpp>
#include <io/transport.hpp>
#include <storage/v3/id_types.hpp>
#include <storage/v3/schemas.hpp>
namespace memgraph::coordinator {
using memgraph::io::Address;
using memgraph::storage::v3::LabelId;
using memgraph::storage::v3::PropertyId;
using memgraph::storage::v3::SchemaProperty;
using SimT = memgraph::io::simulator::SimulatorTransport;
using PrimaryKey = std::vector<PropertyValue>;
struct HlcRequest {
Hlc last_shard_map_version;
};
struct HlcResponse {
Hlc new_hlc;
std::optional<ShardMap> fresher_shard_map;
};
struct GetShardMapRequest {
// No state
};
struct GetShardMapResponse {
ShardMap shard_map;
};
struct AllocateHlcBatchRequest {
Hlc low;
Hlc high;
};
struct AllocateHlcBatchResponse {
bool success;
Hlc low;
Hlc high;
};
struct AllocateEdgeIdBatchRequest {
size_t batch_size;
};
struct AllocateEdgeIdBatchResponse {
uint64_t low;
uint64_t high;
};
struct AllocatePropertyIdsRequest {
std::vector<std::string> property_names;
};
struct AllocatePropertyIdsResponse {
std::map<std::string, PropertyId> property_ids;
};
struct SplitShardRequest {
Hlc previous_shard_map_version;
LabelId label_id;
PrimaryKey split_key;
};
struct SplitShardResponse {
bool success;
};
struct RegisterStorageEngineRequest {
Address address;
};
struct RegisterStorageEngineResponse {
bool success;
};
struct DeregisterStorageEngineRequest {
Address address;
};
struct DeregisterStorageEngineResponse {
bool success;
};
struct InitializeLabelRequest {
std::string label_name;
std::vector<SchemaProperty> schema;
size_t replication_factor;
Hlc last_shard_map_version;
};
struct InitializeLabelResponse {
bool success;
LabelId new_label_id;
std::optional<ShardMap> fresher_shard_map;
};
struct HeartbeatRequest {
Address from_storage_manager;
std::set<boost::uuids::uuid> initialized_rsms;
};
struct HeartbeatResponse {
std::vector<ShardToInitialize> shards_to_initialize;
};
using CoordinatorWriteRequests =
std::variant<HlcRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
DeregisterStorageEngineRequest, InitializeLabelRequest, AllocatePropertyIdsRequest, HeartbeatRequest>;
using CoordinatorWriteResponses = std::variant<HlcResponse, AllocateEdgeIdBatchResponse, SplitShardResponse,
RegisterStorageEngineResponse, DeregisterStorageEngineResponse,
InitializeLabelResponse, AllocatePropertyIdsResponse, HeartbeatResponse>;
using CoordinatorReadRequests = std::variant<GetShardMapRequest>;
using CoordinatorReadResponses = std::variant<GetShardMapResponse>;
class Coordinator {
public:
explicit Coordinator(ShardMap sm) : shard_map_{std::move(sm)} {}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static
CoordinatorReadResponses Read(CoordinatorReadRequests requests) {
return std::visit([&](auto &&request) { return HandleRead(std::forward<decltype(request)>(request)); },
std::move(requests)); // NOLINT(hicpp-move-const-arg,performance-move-const-arg)
}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static
CoordinatorWriteResponses Apply(CoordinatorWriteRequests requests) {
return std::visit([&](auto &&request) mutable { return ApplyWrite(std::forward<decltype(request)>(request)); },
std::move(requests));
}
private:
ShardMap shard_map_;
uint64_t highest_allocated_timestamp_;
/// Query engines need to periodically request batches of unique edge IDs.
uint64_t highest_allocated_edge_id_;
CoordinatorReadResponses HandleRead(GetShardMapRequest && /* get_shard_map_request */) {
GetShardMapResponse res;
res.shard_map = shard_map_;
return res;
}
CoordinatorWriteResponses ApplyWrite(HeartbeatRequest &&heartbeat_request);
CoordinatorWriteResponses ApplyWrite(HlcRequest &&hlc_request);
CoordinatorWriteResponses ApplyWrite(AllocateEdgeIdBatchRequest &&ahr);
/// This splits the shard immediately beneath the provided
/// split key, keeping the assigned peers identical for now,
/// but letting them be gradually migrated over time.
CoordinatorWriteResponses ApplyWrite(SplitShardRequest &&split_shard_request);
/// This adds the provided storage engine to the standby storage engine pool,
/// which can be used to rebalance storage over time.
static CoordinatorWriteResponses ApplyWrite(RegisterStorageEngineRequest && /* register_storage_engine_request */);
/// This begins the process of draining the provided storage engine from all raft
/// clusters that it might be participating in.
static CoordinatorWriteResponses ApplyWrite(DeregisterStorageEngineRequest && /* register_storage_engine_request */);
CoordinatorWriteResponses ApplyWrite(InitializeLabelRequest &&initialize_label_request);
CoordinatorWriteResponses ApplyWrite(AllocatePropertyIdsRequest &&allocate_property_ids_request);
};
} // namespace memgraph::coordinator

View File

@ -0,0 +1,24 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "coordinator/coordinator.hpp"
#include "io/rsm/rsm_client.hpp"
namespace memgraph::coordinator {
using memgraph::io::rsm::RsmClient;
template <typename IoImpl>
using CoordinatorClient = RsmClient<IoImpl, CoordinatorWriteRequests, CoordinatorWriteResponses,
CoordinatorReadRequests, CoordinatorReadResponses>;
} // namespace memgraph::coordinator

View File

@ -0,0 +1,23 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "coordinator/coordinator.hpp"
#include "io/rsm/raft.hpp"
namespace memgraph::coordinator {
template <typename IoImpl>
using CoordinatorRsm = memgraph::io::rsm::Raft<IoImpl, Coordinator, CoordinatorWriteRequests, CoordinatorWriteResponses,
CoordinatorReadRequests, CoordinatorReadResponses>;
} // namespace memgraph::coordinator

View File

@ -0,0 +1,36 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <compare>
#include "io/time.hpp"
namespace memgraph::coordinator {
using Time = memgraph::io::Time;
/// Hybrid-logical clock
struct Hlc {
uint64_t logical_id;
Time coordinator_wall_clock;
auto operator<=>(const Hlc &other) const { return logical_id <=> other.logical_id; }
bool operator==(const Hlc &other) const = default;
bool operator<(const Hlc &other) const = default;
bool operator==(const uint64_t other) const { return logical_id == other; }
bool operator<(const uint64_t other) const { return logical_id < other; }
bool operator>=(const uint64_t other) const { return logical_id >= other; }
};
} // namespace memgraph::coordinator

View File

@ -0,0 +1,90 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "coordinator/shard_map.hpp"
#include "storage/v3/temporal.hpp"
namespace memgraph::coordinator {
using memgraph::common::SchemaType;
using memgraph::storage::v3::TemporalData;
using memgraph::storage::v3::TemporalType;
PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema) {
PrimaryKey ret{};
const int64_t min_int = std::numeric_limits<int64_t>::min();
const TemporalData date{TemporalType::Date, min_int};
const TemporalData local_time{TemporalType::LocalTime, min_int};
const TemporalData local_date_time{TemporalType::LocalDateTime, min_int};
const TemporalData duration{TemporalType::Duration, min_int};
for (const auto &schema_property : schema) {
switch (schema_property.type) {
case SchemaType::BOOL:
ret.emplace_back(PropertyValue(false));
break;
case SchemaType::INT:
ret.emplace_back(PropertyValue(min_int));
break;
case SchemaType::STRING:
ret.emplace_back(PropertyValue(""));
break;
case SchemaType::DATE:
ret.emplace_back(PropertyValue(date));
break;
case SchemaType::LOCALTIME:
ret.emplace_back(PropertyValue(local_time));
break;
case SchemaType::LOCALDATETIME:
ret.emplace_back(PropertyValue(local_date_time));
break;
case SchemaType::DURATION:
ret.emplace_back(PropertyValue(duration));
break;
}
}
return ret;
}
std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
size_t replication_factor, Hlc last_shard_map_version) {
if (shard_map_version != last_shard_map_version || labels.contains(label_name)) {
return std::nullopt;
}
const LabelId label_id = LabelId::FromUint(++max_label_id);
labels.emplace(std::move(label_name), label_id);
PrimaryKey initial_key = SchemaToMinKey(schema);
Shard empty_shard = {};
Shards shards = {
{initial_key, empty_shard},
};
LabelSpace label_space{
.schema = std::move(schema),
.shards = shards,
.replication_factor = replication_factor,
};
label_spaces.emplace(label_id, label_space);
IncrementShardMapVersion();
return label_id;
}
} // namespace memgraph::coordinator

View File

@ -0,0 +1,276 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <limits>
#include <map>
#include <set>
#include <vector>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include "common/types.hpp"
#include "coordinator/hybrid_logical_clock.hpp"
#include "io/address.hpp"
#include "storage/v3/config.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/schemas.hpp"
#include "storage/v3/temporal.hpp"
namespace memgraph::coordinator {
using memgraph::io::Address;
using memgraph::storage::v3::Config;
using memgraph::storage::v3::LabelId;
using memgraph::storage::v3::PropertyId;
using memgraph::storage::v3::PropertyValue;
using memgraph::storage::v3::SchemaProperty;
enum class Status : uint8_t {
CONSENSUS_PARTICIPANT,
INITIALIZING,
// TODO(tyler) this will possibly have more states,
// depending on the reconfiguration protocol that we
// implement.
};
struct AddressAndStatus {
memgraph::io::Address address;
Status status;
friend bool operator<(const AddressAndStatus &lhs, const AddressAndStatus &rhs) { return lhs.address < rhs.address; }
};
using PrimaryKey = std::vector<PropertyValue>;
using Shard = std::vector<AddressAndStatus>;
using Shards = std::map<PrimaryKey, Shard>;
using LabelName = std::string;
using PropertyName = std::string;
using PropertyMap = std::map<PropertyName, PropertyId>;
struct ShardToInitialize {
boost::uuids::uuid uuid;
LabelId label_id;
PrimaryKey min_key;
std::optional<PrimaryKey> max_key;
Config config;
};
PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
struct LabelSpace {
std::vector<SchemaProperty> schema;
std::map<PrimaryKey, Shard> shards;
size_t replication_factor;
};
struct ShardMap {
Hlc shard_map_version;
uint64_t max_property_id;
std::map<PropertyName, PropertyId> properties;
uint64_t max_label_id;
std::map<LabelName, LabelId> labels;
std::map<LabelId, LabelSpace> label_spaces;
std::map<LabelId, std::vector<SchemaProperty>> schemas;
Shards GetShards(const LabelName &label) {
const auto id = labels.at(label);
auto &shards = label_spaces.at(id).shards;
return shards;
}
// TODO(gabor) later we will want to update the wallclock time with
// the given Io<impl>'s time as well
Hlc IncrementShardMapVersion() noexcept {
++shard_map_version.logical_id;
return shard_map_version;
}
Hlc GetHlc() const noexcept { return shard_map_version; }
// Returns the shard UUIDs that have been assigned but not yet acknowledged for this storage manager
std::vector<ShardToInitialize> AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized) {
std::vector<ShardToInitialize> ret{};
bool mutated = false;
for (auto &[label_id, label_space] : label_spaces) {
for (auto &[low_key, shard] : label_space.shards) {
// TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
bool machine_contains_shard = false;
for (auto &aas : shard) {
if (initialized.contains(aas.address.unique_id)) {
spdlog::info("marking shard as full consensus participant: {}", aas.address.unique_id);
aas.status = Status::CONSENSUS_PARTICIPANT;
machine_contains_shard = true;
} else {
const bool same_machine = aas.address.last_known_ip == storage_manager.last_known_ip &&
aas.address.last_known_port == storage_manager.last_known_port;
if (same_machine) {
machine_contains_shard = true;
ret.push_back(ShardToInitialize{
.uuid = aas.address.unique_id,
.label_id = label_id,
.min_key = low_key,
.max_key = std::nullopt,
.config = Config{},
});
}
}
}
if (!machine_contains_shard && shard.size() < label_space.replication_factor) {
Address address = storage_manager;
// TODO(tyler) use deterministic UUID so that coordinators don't diverge here
address.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
ret.push_back(ShardToInitialize{
.uuid = address.unique_id,
.label_id = label_id,
.min_key = low_key,
.max_key = std::nullopt,
.config = Config{},
});
AddressAndStatus aas = {
.address = address,
.status = Status::INITIALIZING,
};
shard.emplace_back(aas);
}
}
}
if (mutated) {
IncrementShardMapVersion();
}
return ret;
}
bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key) {
if (previous_shard_map_version != shard_map_version) {
return false;
}
auto &label_space = label_spaces.at(label_id);
auto &shards_in_map = label_space.shards;
MG_ASSERT(!shards_in_map.empty());
MG_ASSERT(!shards_in_map.contains(key));
MG_ASSERT(label_spaces.contains(label_id));
// Finding the Shard that the new PrimaryKey should map to.
auto prev = std::prev(shards_in_map.upper_bound(key));
Shard duplicated_shard = prev->second;
// Apply the split
shards_in_map[key] = duplicated_shard;
return true;
}
std::optional<LabelId> InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
size_t replication_factor, Hlc last_shard_map_version);
void AddServer(Address server_address) {
// Find a random place for the server to plug in
}
LabelId GetLabelId(const std::string &label) const { return labels.at(label); }
Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const {
MG_ASSERT(start_key <= end_key);
MG_ASSERT(labels.contains(label_name));
LabelId label_id = labels.at(label_name);
const auto &label_space = label_spaces.at(label_id);
const auto &shards_for_label = label_space.shards;
MG_ASSERT(shards_for_label.begin()->first <= start_key,
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
auto it = std::prev(shards_for_label.upper_bound(start_key));
const auto end_it = shards_for_label.upper_bound(end_key);
Shards shards{};
std::copy(it, end_it, std::inserter(shards, shards.end()));
return shards;
}
Shard GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
MG_ASSERT(labels.contains(label_name));
LabelId label_id = labels.at(label_name);
const auto &label_space = label_spaces.at(label_id);
MG_ASSERT(label_space.shards.begin()->first <= key,
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
return std::prev(label_space.shards.upper_bound(key))->second;
}
Shard GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
MG_ASSERT(label_spaces.contains(label_id));
const auto &label_space = label_spaces.at(label_id);
MG_ASSERT(label_space.shards.begin()->first <= key,
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
return std::prev(label_space.shards.upper_bound(key))->second;
}
PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties) {
PropertyMap ret{};
bool mutated = false;
for (const auto &property_name : new_properties) {
if (properties.contains(property_name)) {
auto property_id = properties.at(property_name);
ret.emplace(property_name, property_id);
} else {
mutated = true;
const PropertyId property_id = PropertyId::FromUint(++max_property_id);
ret.emplace(property_name, property_id);
properties.emplace(property_name, property_id);
}
}
if (mutated) {
IncrementShardMapVersion();
}
return ret;
}
std::optional<PropertyId> GetPropertyId(const std::string &property_name) const {
if (properties.contains(property_name)) {
return properties.at(property_name);
}
return std::nullopt;
}
};
} // namespace memgraph::coordinator

20
src/expr/CMakeLists.txt Normal file
View File

@ -0,0 +1,20 @@
define_add_lcp(add_lcp_expr lcp_expr_cpp_files generated_lcp_expr_files)
add_lcp_expr(semantic/symbol.lcp)
add_custom_target(generate_lcp_expr DEPENDS ${generated_lcp_expr_files})
set(mg_expr_sources
${lcp_expr_cpp_files}
parsing.cpp)
find_package(Boost REQUIRED)
add_library(mg-expr STATIC ${mg_expr_sources})
add_dependencies(mg-expr generate_lcp_expr)
target_include_directories(mg-expr PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/ast)
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/interpret)
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/semantic)
target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser)

35
src/expr/ast.hpp Normal file
View File

@ -0,0 +1,35 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifndef MG_AST_INCLUDE_PATH
#ifdef MG_CLANG_TIDY_CHECK
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define MG_AST_INCLUDE_PATH "query/v2/frontend/ast/ast.hpp"
#else
#error Missing AST include path
#endif
#endif
#ifndef MG_INJECTED_NAMESPACE_NAME
#ifdef MG_CLANG_TIDY_CHECK
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define MG_INJECTED_NAMESPACE_NAME memgraph::query::v2
#else
#error Missing AST namespace
#endif
#endif
#include MG_AST_INCLUDE_PATH
namespace memgraph::expr {
using namespace MG_INJECTED_NAMESPACE_NAME; // NOLINT(google-build-using-namespace)
} // namespace memgraph::expr

View File

@ -0,0 +1,135 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "utils/visitor.hpp"
namespace MG_INJECTED_NAMESPACE_NAME {
// Forward declares for Tree visitors.
class CypherQuery;
class SingleQuery;
class CypherUnion;
class NamedExpression;
class Identifier;
class PropertyLookup;
class LabelsTest;
class Aggregation;
class Function;
class Reduce;
class Coalesce;
class Extract;
class All;
class Single;
class Any;
class None;
class ParameterLookup;
class CallProcedure;
class Create;
class Match;
class Return;
class With;
class Pattern;
class NodeAtom;
class EdgeAtom;
class PrimitiveLiteral;
class ListLiteral;
class MapLiteral;
class OrOperator;
class XorOperator;
class AndOperator;
class NotOperator;
class AdditionOperator;
class SubtractionOperator;
class MultiplicationOperator;
class DivisionOperator;
class ModOperator;
class UnaryPlusOperator;
class UnaryMinusOperator;
class IsNullOperator;
class NotEqualOperator;
class EqualOperator;
class LessOperator;
class GreaterOperator;
class LessEqualOperator;
class GreaterEqualOperator;
class InListOperator;
class SubscriptOperator;
class ListSlicingOperator;
class IfOperator;
class Delete;
class Where;
class SetProperty;
class SetProperties;
class SetLabels;
class RemoveProperty;
class RemoveLabels;
class Merge;
class Unwind;
class AuthQuery;
class ExplainQuery;
class ProfileQuery;
class IndexQuery;
class InfoQuery;
class ConstraintQuery;
class RegexMatch;
class DumpQuery;
class ReplicationQuery;
class LockPathQuery;
class LoadCsv;
class FreeMemoryQuery;
class TriggerQuery;
class IsolationLevelQuery;
class CreateSnapshotQuery;
class StreamQuery;
class SettingQuery;
class VersionQuery;
class Foreach;
class SchemaQuery;
using TreeCompositeVisitor = memgraph::utils::CompositeVisitor<
SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
SubtractionOperator, MultiplicationOperator, DivisionOperator, ModOperator, NotEqualOperator, EqualOperator,
LessOperator, GreaterOperator, LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator,
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral, MapLiteral,
PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any, None, CallProcedure,
Create, Match, Return, With, Pattern, NodeAtom, EdgeAtom, Delete, Where, SetProperty, SetProperties, SetLabels,
RemoveProperty, RemoveLabels, Merge, Unwind, RegexMatch, LoadCsv, Foreach>;
using TreeLeafVisitor = memgraph::utils::LeafVisitor<Identifier, PrimitiveLiteral, ParameterLookup>;
class HierarchicalTreeVisitor : public TreeCompositeVisitor, public TreeLeafVisitor {
public:
using TreeCompositeVisitor::PostVisit;
using TreeCompositeVisitor::PreVisit;
using TreeLeafVisitor::Visit;
using typename TreeLeafVisitor::ReturnType;
};
template <class TResult>
class ExpressionVisitor
: public memgraph::utils::Visitor<
TResult, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
SubtractionOperator, MultiplicationOperator, DivisionOperator, ModOperator, NotEqualOperator, EqualOperator,
LessOperator, GreaterOperator, LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator,
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral,
MapLiteral, PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any,
None, ParameterLookup, Identifier, PrimitiveLiteral, RegexMatch> {};
template <class TResult>
class QueryVisitor
: public memgraph::utils::Visitor<TResult, CypherQuery, ExplainQuery, ProfileQuery, IndexQuery, AuthQuery,
InfoQuery, ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery,
FreeMemoryQuery, TriggerQuery, IsolationLevelQuery, CreateSnapshotQuery,
StreamQuery, SettingQuery, VersionQuery, SchemaQuery> {};
} // namespace MG_INJECTED_NAMESPACE_NAME

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,271 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <iostream>
#include <type_traits>
#include "expr/ast.hpp"
#include "expr/typed_value.hpp"
#include "utils/algorithm.hpp"
#include "utils/logging.hpp"
#include "utils/string.hpp"
namespace memgraph::expr {
namespace detail {
template <typename T>
void PrintObject(std::ostream *out, const T &arg) {
static_assert(!std::is_convertible<T, Expression *>::value,
"This overload shouldn't be called with pointers convertible "
"to Expression *. This means your other PrintObject overloads aren't "
"being called for certain AST nodes when they should (or perhaps such "
"overloads don't exist yet).");
*out << arg;
}
inline void PrintObject(std::ostream *out, const std::string &str) { *out << utils::Escape(str); }
inline void PrintObject(std::ostream *out, Aggregation::Op op) { *out << Aggregation::OpToString(op); }
inline void PrintObject(std::ostream *out, Expression *expr);
inline void PrintObject(std::ostream *out, Identifier *expr) { PrintObject(out, static_cast<Expression *>(expr)); }
template <typename T>
void PrintObject(std::ostream *out, const std::vector<T> &vec) {
*out << "[";
utils::PrintIterable(*out, vec, ", ", [](auto &stream, const auto &item) { PrintObject(&stream, item); });
*out << "]";
}
template <typename T>
void PrintObject(std::ostream *out, const std::vector<T, utils::Allocator<T>> &vec) {
*out << "[";
utils::PrintIterable(*out, vec, ", ", [](auto &stream, const auto &item) { PrintObject(&stream, item); });
*out << "]";
}
template <typename K, typename V>
void PrintObject(std::ostream *out, const std::map<K, V> &map) {
*out << "{";
utils::PrintIterable(*out, map, ", ", [](auto &stream, const auto &item) {
PrintObject(&stream, item.first);
stream << ": ";
PrintObject(&stream, item.second);
});
*out << "}";
}
template <typename T>
void PrintObject(std::ostream *out, const utils::pmr::map<utils::pmr::string, T> &map) {
*out << "{";
utils::PrintIterable(*out, map, ", ", [](auto &stream, const auto &item) {
PrintObject(&stream, item.first);
stream << ": ";
PrintObject(&stream, item.second);
});
*out << "}";
}
template <typename T1, typename T2, typename T3>
inline void PrintObject(std::ostream *out, const TypedValueT<T1, T2, T3> &value) {
using TypedValue = TypedValueT<T1, T2, T3>;
switch (value.type()) {
case TypedValue::Type::Null:
*out << "null";
break;
case TypedValue::Type::String:
PrintObject(out, value.ValueString());
break;
case TypedValue::Type::Bool:
*out << (value.ValueBool() ? "true" : "false");
break;
case TypedValue::Type::Int:
PrintObject(out, value.ValueInt());
break;
case TypedValue::Type::Double:
PrintObject(out, value.ValueDouble());
break;
case TypedValue::Type::List:
PrintObject(out, value.ValueList());
break;
case TypedValue::Type::Map:
PrintObject(out, value.ValueMap());
break;
case TypedValue::Type::Date:
PrintObject(out, value.ValueDate());
break;
case TypedValue::Type::Duration:
PrintObject(out, value.ValueDuration());
break;
case TypedValue::Type::LocalTime:
PrintObject(out, value.ValueLocalTime());
break;
case TypedValue::Type::LocalDateTime:
PrintObject(out, value.ValueLocalDateTime());
break;
default:
MG_ASSERT(false, "PrintObject(std::ostream *out, const TypedValue &value) should not reach here");
}
}
template <typename T>
void PrintOperatorArgs(std::ostream *out, const T &arg) {
*out << " ";
PrintObject(out, arg);
*out << ")";
}
template <typename T, typename... Ts>
void PrintOperatorArgs(std::ostream *out, const T &arg, const Ts &...args) {
*out << " ";
PrintObject(out, arg);
PrintOperatorArgs(out, args...);
}
template <typename... Ts>
void PrintOperator(std::ostream *out, const std::string &name, const Ts &...args) {
*out << "(" << name;
PrintOperatorArgs(out, args...);
}
} // namespace detail
class ExpressionPrettyPrinter : public ExpressionVisitor<void> {
public:
explicit ExpressionPrettyPrinter(std::ostream *out) : out_(out) {}
// Unary operators
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define UNARY_OPERATOR_VISIT(OP_NODE, OP_STR) \
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
void Visit(OP_NODE &op) override { detail::PrintOperator(out_, OP_STR, op.expression_); }
UNARY_OPERATOR_VISIT(NotOperator, "Not");
UNARY_OPERATOR_VISIT(UnaryPlusOperator, "+");
UNARY_OPERATOR_VISIT(UnaryMinusOperator, "-");
UNARY_OPERATOR_VISIT(IsNullOperator, "IsNull");
#undef UNARY_OPERATOR_VISIT
// Binary operators
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define BINARY_OPERATOR_VISIT(OP_NODE, OP_STR) \
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
void Visit(OP_NODE &op) override { detail::PrintOperator(out_, OP_STR, op.expression1_, op.expression2_); }
BINARY_OPERATOR_VISIT(OrOperator, "Or");
BINARY_OPERATOR_VISIT(XorOperator, "Xor");
BINARY_OPERATOR_VISIT(AndOperator, "And");
BINARY_OPERATOR_VISIT(AdditionOperator, "+");
BINARY_OPERATOR_VISIT(SubtractionOperator, "-");
BINARY_OPERATOR_VISIT(MultiplicationOperator, "*");
BINARY_OPERATOR_VISIT(DivisionOperator, "/");
BINARY_OPERATOR_VISIT(ModOperator, "%");
BINARY_OPERATOR_VISIT(NotEqualOperator, "!=");
BINARY_OPERATOR_VISIT(EqualOperator, "==");
BINARY_OPERATOR_VISIT(LessOperator, "<");
BINARY_OPERATOR_VISIT(GreaterOperator, ">");
BINARY_OPERATOR_VISIT(LessEqualOperator, "<=");
BINARY_OPERATOR_VISIT(GreaterEqualOperator, ">=");
BINARY_OPERATOR_VISIT(InListOperator, "In");
BINARY_OPERATOR_VISIT(SubscriptOperator, "Subscript");
#undef BINARY_OPERATOR_VISIT
// Other
void Visit(ListSlicingOperator &op) override {
detail::PrintOperator(out_, "ListSlicing", op.list_, op.lower_bound_, op.upper_bound_);
}
void Visit(IfOperator &op) override {
detail::PrintOperator(out_, "If", op.condition_, op.then_expression_, op.else_expression_);
}
void Visit(ListLiteral &op) override { detail::PrintOperator(out_, "ListLiteral", op.elements_); }
void Visit(MapLiteral &op) override {
std::map<std::string, Expression *> map;
for (const auto &kv : op.elements_) {
map[kv.first.name] = kv.second;
}
detail::PrintObject(out_, map);
}
void Visit(LabelsTest &op) override { detail::PrintOperator(out_, "LabelsTest", op.expression_); }
void Visit(Aggregation &op) override { detail::PrintOperator(out_, "Aggregation", op.op_); }
void Visit(Function &op) override { detail::PrintOperator(out_, "Function", op.function_name_, op.arguments_); }
void Visit(Reduce &op) override {
detail::PrintOperator(out_, "Reduce", op.accumulator_, op.initializer_, op.identifier_, op.list_, op.expression_);
}
void Visit(Coalesce &op) override { detail::PrintOperator(out_, "Coalesce", op.expressions_); }
void Visit(Extract &op) override { detail::PrintOperator(out_, "Extract", op.identifier_, op.list_, op.expression_); }
void Visit(All &op) override {
detail::PrintOperator(out_, "All", op.identifier_, op.list_expression_, op.where_->expression_);
}
void Visit(Single &op) override {
detail::PrintOperator(out_, "Single", op.identifier_, op.list_expression_, op.where_->expression_);
}
void Visit(Any &op) override {
detail::PrintOperator(out_, "Any", op.identifier_, op.list_expression_, op.where_->expression_);
}
void Visit(None &op) override {
detail::PrintOperator(out_, "None", op.identifier_, op.list_expression_, op.where_->expression_);
}
void Visit(Identifier &op) override { detail::PrintOperator(out_, "Identifier", op.name_); }
void Visit(PrimitiveLiteral &op) override { detail::PrintObject(out_, op.value_); }
void Visit(PropertyLookup &op) override {
detail::PrintOperator(out_, "PropertyLookup", op.expression_, op.property_.name);
}
void Visit(ParameterLookup &op) override { detail::PrintOperator(out_, "ParameterLookup", op.token_position_); }
void Visit(NamedExpression &op) override { detail::PrintOperator(out_, "NamedExpression", op.name_, op.expression_); }
void Visit(RegexMatch &op) override { detail::PrintOperator(out_, "=~", op.string_expr_, op.regex_); }
private:
std::ostream *out_;
};
namespace detail {
inline void PrintObject(std::ostream *out, Expression *expr) {
if (expr) {
ExpressionPrettyPrinter printer{out};
expr->Accept(printer);
} else {
*out << "<null>";
}
}
} // namespace detail
inline void PrintExpression(Expression *expr, std::ostream *out) {
ExpressionPrettyPrinter printer{out};
expr->Accept(printer);
}
inline void PrintExpression(NamedExpression *expr, std::ostream *out) {
ExpressionPrettyPrinter printer{out};
expr->Accept(printer);
}
} // namespace memgraph::expr

52
src/expr/exceptions.hpp Normal file
View File

@ -0,0 +1,52 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "utils/exceptions.hpp"
namespace memgraph::expr {
class SyntaxException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SyntaxException() : SyntaxException("") {}
};
class SemanticException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SemanticException() : BasicException("") {}
};
class ExpressionRuntimeException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
};
class RedeclareVariableError : public SemanticException {
public:
explicit RedeclareVariableError(const std::string &name) : SemanticException("Redeclaring variable: " + name + ".") {}
};
class UnboundVariableError : public SemanticException {
public:
explicit UnboundVariableError(const std::string &name) : SemanticException("Unbound variable: " + name + ".") {}
};
class TypeMismatchError : public SemanticException {
public:
TypeMismatchError(const std::string &name, const std::string &datum, const std::string &expected)
: SemanticException(fmt::format("Type mismatch: {} already defined as {}, expected {}.", name, datum, expected)) {
}
};
} // namespace memgraph::expr

826
src/expr/interpret/eval.hpp Normal file
View File

@ -0,0 +1,826 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file
#pragma once
#include <algorithm>
#include <limits>
#include <map>
#include <optional>
#include <regex>
#include <type_traits>
#include <vector>
#include "expr/ast.hpp"
#include "expr/exceptions.hpp"
#include "expr/interpret/frame.hpp"
#include "expr/semantic/symbol_table.hpp"
#include "utils/exceptions.hpp"
namespace memgraph::expr {
struct StorageTag {};
struct QueryEngineTag {};
template <typename TypedValue, typename EvaluationContext, typename DbAccessor, typename StorageView, typename LabelId,
typename PropertyValue, typename ConvFunctor, typename Error, typename Tag = StorageTag>
class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
public:
ExpressionEvaluator(Frame<TypedValue> *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx,
DbAccessor *dba, StorageView view)
: frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {}
using ExpressionVisitor<TypedValue>::Visit;
utils::MemoryResource *GetMemoryResource() const { return ctx_->memory; }
TypedValue Visit(NamedExpression &named_expression) override {
const auto &symbol = symbol_table_->at(named_expression);
auto value = named_expression.expression_->Accept(*this);
frame_->at(symbol) = value;
return value;
}
TypedValue Visit(Identifier &ident) override {
return TypedValue(frame_->at(symbol_table_->at(ident)), ctx_->memory);
}
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define BINARY_OPERATOR_VISITOR(OP_NODE, CPP_OP, CYPHER_OP) \
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
TypedValue Visit(OP_NODE &op) override { \
auto val1 = op.expression1_->Accept(*this); \
auto val2 = op.expression2_->Accept(*this); \
try { \
return val1 CPP_OP val2; \
} catch (const TypedValueException &) { \
throw ExpressionRuntimeException("Invalid types: {} and {} for '{}'.", val1.type(), val2.type(), #CYPHER_OP); \
} \
}
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define UNARY_OPERATOR_VISITOR(OP_NODE, CPP_OP, CYPHER_OP) \
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
TypedValue Visit(OP_NODE &op) override { \
auto val = op.expression_->Accept(*this); \
try { \
return CPP_OP val; \
} catch (const TypedValueException &) { \
throw ExpressionRuntimeException("Invalid type {} for '{}'.", val.type(), #CYPHER_OP); \
} \
}
BINARY_OPERATOR_VISITOR(OrOperator, ||, OR);
BINARY_OPERATOR_VISITOR(XorOperator, ^, XOR);
BINARY_OPERATOR_VISITOR(AdditionOperator, +, +);
BINARY_OPERATOR_VISITOR(SubtractionOperator, -, -);
BINARY_OPERATOR_VISITOR(MultiplicationOperator, *, *);
BINARY_OPERATOR_VISITOR(DivisionOperator, /, /);
BINARY_OPERATOR_VISITOR(ModOperator, %, %);
BINARY_OPERATOR_VISITOR(NotEqualOperator, !=, <>);
BINARY_OPERATOR_VISITOR(EqualOperator, ==, =);
BINARY_OPERATOR_VISITOR(LessOperator, <, <);
BINARY_OPERATOR_VISITOR(GreaterOperator, >, >);
BINARY_OPERATOR_VISITOR(LessEqualOperator, <=, <=);
BINARY_OPERATOR_VISITOR(GreaterEqualOperator, >=, >=);
UNARY_OPERATOR_VISITOR(NotOperator, !, NOT);
UNARY_OPERATOR_VISITOR(UnaryPlusOperator, +, +);
UNARY_OPERATOR_VISITOR(UnaryMinusOperator, -, -);
#undef BINARY_OPERATOR_VISITOR
#undef UNARY_OPERATOR_VISITOR
TypedValue Visit(AndOperator &op) override {
auto value1 = op.expression1_->Accept(*this);
if (value1.IsBool() && !value1.ValueBool()) {
// If first expression is false, don't evaluate the second one.
return value1;
}
auto value2 = op.expression2_->Accept(*this);
try {
return value1 && value2;
} catch (const TypedValueException &) {
throw ExpressionRuntimeException("Invalid types: {} and {} for AND.", value1.type(), value2.type());
}
}
TypedValue Visit(IfOperator &if_operator) override {
auto condition = if_operator.condition_->Accept(*this);
if (condition.IsNull()) {
return if_operator.else_expression_->Accept(*this);
}
if (condition.type() != TypedValue::Type::Bool) {
// At the moment IfOperator is used only in CASE construct.
throw ExpressionRuntimeException("CASE expected boolean expression, got {}.", condition.type());
}
if (condition.ValueBool()) {
return if_operator.then_expression_->Accept(*this);
}
return if_operator.else_expression_->Accept(*this);
}
TypedValue Visit(InListOperator &in_list) override {
auto literal = in_list.expression1_->Accept(*this);
auto _list = in_list.expression2_->Accept(*this);
if (_list.IsNull()) {
return TypedValue(ctx_->memory);
}
// Exceptions have higher priority than returning nulls when list expression
// is not null.
if (_list.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("IN expected a list, got {}.", _list.type());
}
const auto &list = _list.ValueList();
// If literal is NULL there is no need to try to compare it with every
// element in the list since result of every comparison will be NULL. There
// is one special case that we must test explicitly: if list is empty then
// result is false since no comparison will be performed.
if (list.empty()) return TypedValue(false, ctx_->memory);
if (literal.IsNull()) return TypedValue(ctx_->memory);
auto has_null = false;
for (const auto &element : list) {
auto result = literal == element;
if (result.IsNull()) {
has_null = true;
} else if (result.ValueBool()) {
return TypedValue(true, ctx_->memory);
}
}
if (has_null) {
return TypedValue(ctx_->memory);
}
return TypedValue(false, ctx_->memory);
}
TypedValue Visit(SubscriptOperator &list_indexing) override {
auto lhs = list_indexing.expression1_->Accept(*this);
auto index = list_indexing.expression2_->Accept(*this);
if (!lhs.IsList() && !lhs.IsMap() && !lhs.IsVertex() && !lhs.IsEdge() && !lhs.IsNull())
throw ExpressionRuntimeException(
"Expected a list, a map, a node or an edge to index with '[]', got "
"{}.",
lhs.type());
if (lhs.IsNull() || index.IsNull()) return TypedValue(ctx_->memory);
if (lhs.IsList()) {
if (!index.IsInt())
throw ExpressionRuntimeException("Expected an integer as a list index, got {}.", index.type());
auto index_int = index.ValueInt();
// NOTE: Take non-const reference to list, so that we can move out the
// indexed element as the result.
auto &list = lhs.ValueList();
if (index_int < 0) {
index_int += static_cast<int64_t>(list.size());
}
if (index_int >= static_cast<int64_t>(list.size()) || index_int < 0) return TypedValue(ctx_->memory);
// NOTE: Explicit move is needed, so that we return the move constructed
// value and preserve the correct MemoryResource.
return std::move(list[index_int]);
}
if (lhs.IsMap()) {
if (!index.IsString())
throw ExpressionRuntimeException("Expected a string as a map index, got {}.", index.type());
// NOTE: Take non-const reference to map, so that we can move out the
// looked-up element as the result.
auto &map = lhs.ValueMap();
auto found = map.find(index.ValueString());
if (found == map.end()) return TypedValue(ctx_->memory);
// NOTE: Explicit move is needed, so that we return the move constructed
// value and preserve the correct MemoryResource.
return std::move(found->second);
}
if (lhs.IsVertex()) {
if (!index.IsString())
throw ExpressionRuntimeException("Expected a string as a property name, got {}.", index.type());
return TypedValue(GetProperty(lhs.ValueVertex(), index.ValueString()), ctx_->memory);
}
if (lhs.IsEdge()) {
if (!index.IsString())
throw ExpressionRuntimeException("Expected a string as a property name, got {}.", index.type());
return TypedValue(GetProperty(lhs.ValueEdge(), index.ValueString()), ctx_->memory);
}
// lhs is Null
return TypedValue(ctx_->memory);
}
TypedValue Visit(ListSlicingOperator &op) override {
// If some type is null we can't return null, because throwing exception
// on illegal type has higher priority.
auto is_null = false;
auto get_bound = [&](Expression *bound_expr, int64_t default_value) {
if (bound_expr) {
auto bound = bound_expr->Accept(*this);
if (bound.type() == TypedValue::Type::Null) {
is_null = true;
} else if (bound.type() != TypedValue::Type::Int) {
throw ExpressionRuntimeException("Expected an integer for a bound in list slicing, got {}.", bound.type());
}
return bound;
}
return TypedValue(default_value, ctx_->memory);
};
auto _upper_bound = get_bound(op.upper_bound_, std::numeric_limits<int64_t>::max());
auto _lower_bound = get_bound(op.lower_bound_, 0);
auto _list = op.list_->Accept(*this);
if (_list.type() == TypedValue::Type::Null) {
is_null = true;
} else if (_list.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("Expected a list to slice, got {}.", _list.type());
}
if (is_null) {
return TypedValue(ctx_->memory);
}
const auto &list = _list.ValueList();
auto normalise_bound = [&](int64_t bound) {
if (bound < 0) {
bound = static_cast<int64_t>(list.size()) + bound;
}
return std::max(static_cast<int64_t>(0), std::min(bound, static_cast<int64_t>(list.size())));
};
auto lower_bound = normalise_bound(_lower_bound.ValueInt());
auto upper_bound = normalise_bound(_upper_bound.ValueInt());
if (upper_bound <= lower_bound) {
return TypedValue(typename TypedValue::TVector(ctx_->memory), ctx_->memory);
}
return TypedValue(
typename TypedValue::TVector(list.begin() + lower_bound, list.begin() + upper_bound, ctx_->memory));
}
TypedValue Visit(IsNullOperator &is_null) override {
auto value = is_null.expression_->Accept(*this);
return TypedValue(value.IsNull(), ctx_->memory);
}
TypedValue Visit(PropertyLookup &property_lookup) override {
auto expression_result = property_lookup.expression_->Accept(*this);
auto maybe_date = [this](const auto &date, const auto &prop_name) -> std::optional<TypedValue> {
if (prop_name == "year") {
return TypedValue(date.year, ctx_->memory);
}
if (prop_name == "month") {
return TypedValue(date.month, ctx_->memory);
}
if (prop_name == "day") {
return TypedValue(date.day, ctx_->memory);
}
return std::nullopt;
};
auto maybe_local_time = [this](const auto &lt, const auto &prop_name) -> std::optional<TypedValue> {
if (prop_name == "hour") {
return TypedValue(lt.hour, ctx_->memory);
}
if (prop_name == "minute") {
return TypedValue(lt.minute, ctx_->memory);
}
if (prop_name == "second") {
return TypedValue(lt.second, ctx_->memory);
}
if (prop_name == "millisecond") {
return TypedValue(lt.millisecond, ctx_->memory);
}
if (prop_name == "microsecond") {
return TypedValue(lt.microsecond, ctx_->memory);
}
return std::nullopt;
};
auto maybe_duration = [this](const auto &dur, const auto &prop_name) -> std::optional<TypedValue> {
if (prop_name == "day") {
return TypedValue(dur.Days(), ctx_->memory);
}
if (prop_name == "hour") {
return TypedValue(dur.SubDaysAsHours(), ctx_->memory);
}
if (prop_name == "minute") {
return TypedValue(dur.SubDaysAsMinutes(), ctx_->memory);
}
if (prop_name == "second") {
return TypedValue(dur.SubDaysAsSeconds(), ctx_->memory);
}
if (prop_name == "millisecond") {
return TypedValue(dur.SubDaysAsMilliseconds(), ctx_->memory);
}
if (prop_name == "microsecond") {
return TypedValue(dur.SubDaysAsMicroseconds(), ctx_->memory);
}
if (prop_name == "nanosecond") {
return TypedValue(dur.SubDaysAsNanoseconds(), ctx_->memory);
}
return std::nullopt;
};
switch (expression_result.type()) {
case TypedValue::Type::Null:
return TypedValue(ctx_->memory);
case TypedValue::Type::Vertex:
return GetProperty(expression_result.ValueVertex(), property_lookup.property_);
case TypedValue::Type::Edge:
return GetProperty(expression_result.ValueEdge(), property_lookup.property_);
case TypedValue::Type::Map: {
// NOTE: Take non-const reference to map, so that we can move out the
// looked-up element as the result.
auto &map = expression_result.ValueMap();
auto found = map.find(property_lookup.property_.name.c_str());
if (found == map.end()) return TypedValue(ctx_->memory);
// NOTE: Explicit move is needed, so that we return the move constructed
// value and preserve the correct MemoryResource.
return std::move(found->second);
}
case TypedValue::Type::Duration: {
const auto &prop_name = property_lookup.property_.name;
const auto &dur = expression_result.ValueDuration();
if (auto dur_field = maybe_duration(dur, prop_name); dur_field) {
return std::move(*dur_field);
}
throw ExpressionRuntimeException("Invalid property name {} for Duration", prop_name);
}
case TypedValue::Type::Date: {
const auto &prop_name = property_lookup.property_.name;
const auto &date = expression_result.ValueDate();
if (auto date_field = maybe_date(date, prop_name); date_field) {
return std::move(*date_field);
}
throw ExpressionRuntimeException("Invalid property name {} for Date", prop_name);
}
case TypedValue::Type::LocalTime: {
const auto &prop_name = property_lookup.property_.name;
const auto &lt = expression_result.ValueLocalTime();
if (auto lt_field = maybe_local_time(lt, prop_name); lt_field) {
return std::move(*lt_field);
}
throw ExpressionRuntimeException("Invalid property name {} for LocalTime", prop_name);
}
case TypedValue::Type::LocalDateTime: {
const auto &prop_name = property_lookup.property_.name;
const auto &ldt = expression_result.ValueLocalDateTime();
if (auto date_field = maybe_date(ldt.date, prop_name); date_field) {
return std::move(*date_field);
}
if (auto lt_field = maybe_local_time(ldt.local_time, prop_name); lt_field) {
return std::move(*lt_field);
}
throw ExpressionRuntimeException("Invalid property name {} for LocalDateTime", prop_name);
}
default:
throw ExpressionRuntimeException("Only nodes, edges, maps and temporal types have properties to be looked-up.");
}
}
template <typename VertexAccessor, typename TTag = Tag,
typename TReturnType = std::enable_if_t<std::is_same_v<TTag, StorageTag>, bool>>
TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label, StorageTag /*tag*/) {
auto has_label = vertex.HasLabel(view_, GetLabel(label));
if (has_label.HasError() && has_label.GetError() == Error::NONEXISTENT_OBJECT) {
// This is a very nasty and temporary hack in order to make MERGE
// work. The old storage had the following logic when returning an
// `OLD` view: `return old ? old : new`. That means that if the
// `OLD` view didn't exist, it returned the NEW view. With this hack
// we simulate that behavior.
// TODO (mferencevic, teon.banek): Remove once MERGE is
// reimplemented.
has_label = vertex.HasLabel(StorageView::NEW, GetLabel(label));
}
if (has_label.HasError()) {
switch (has_label.GetError()) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to access labels on a deleted node.");
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to access labels from a node that doesn't exist.");
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
throw ExpressionRuntimeException("Unexpected error when accessing labels.");
}
}
return *has_label;
}
template <typename VertexAccessor, typename TTag = Tag,
typename TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, bool>>
TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label_ix, QueryEngineTag /*tag*/) {
auto label = typename VertexAccessor::Label{LabelId::FromUint(label_ix.ix)};
auto has_label = vertex.HasLabel(label);
return !has_label;
}
TypedValue Visit(LabelsTest &labels_test) override {
auto expression_result = labels_test.expression_->Accept(*this);
switch (expression_result.type()) {
case TypedValue::Type::Null:
return TypedValue(ctx_->memory);
case TypedValue::Type::Vertex: {
const auto &vertex = expression_result.ValueVertex();
if (std::ranges::all_of(labels_test.labels_, [&vertex, this](const auto label_test) {
return this->HasLabelImpl(vertex, label_test, Tag{});
})) {
return TypedValue(true, ctx_->memory);
}
return TypedValue(false, ctx_->memory);
}
default:
throw ExpressionRuntimeException("Only nodes have labels.");
}
}
TypedValue Visit(PrimitiveLiteral &literal) override {
// TODO: no need to evaluate constants, we can write it to frame in one
// of the previous phases.
return TypedValue(literal.value_, ctx_->memory);
}
TypedValue Visit(ListLiteral &literal) override {
typename TypedValue::TVector result(ctx_->memory);
result.reserve(literal.elements_.size());
for (const auto &expression : literal.elements_) result.emplace_back(expression->Accept(*this));
return TypedValue(result, ctx_->memory);
}
TypedValue Visit(MapLiteral &literal) override {
typename TypedValue::TMap result(ctx_->memory);
for (const auto &pair : literal.elements_) result.emplace(pair.first.name, pair.second->Accept(*this));
return TypedValue(result, ctx_->memory);
}
TypedValue Visit(Aggregation &aggregation) override {
return TypedValue(frame_->at(symbol_table_->at(aggregation)), ctx_->memory);
}
TypedValue Visit(Coalesce &coalesce) override {
auto &exprs = coalesce.expressions_;
if (exprs.size() == 0) {
throw ExpressionRuntimeException("'coalesce' requires at least one argument.");
}
for (int64_t i = 0; i < exprs.size(); ++i) {
TypedValue val(exprs[i]->Accept(*this), ctx_->memory);
if (!val.IsNull()) {
return val;
}
}
return TypedValue(ctx_->memory);
}
TypedValue Visit(Function &function) override {
FunctionContext function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
// Stack allocate evaluated arguments when there's a small number of them.
if (function.arguments_.size() <= 8) {
TypedValue arguments[8] = {TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory),
TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory),
TypedValue(ctx_->memory), TypedValue(ctx_->memory)};
for (size_t i = 0; i < function.arguments_.size(); ++i) {
arguments[i] = function.arguments_[i]->Accept(*this);
}
auto res = function.function_(arguments, function.arguments_.size(), function_ctx);
MG_ASSERT(res.GetMemoryResource() == ctx_->memory);
return res;
} else {
typename TypedValue::TVector arguments(ctx_->memory);
arguments.reserve(function.arguments_.size());
for (const auto &argument : function.arguments_) {
arguments.emplace_back(argument->Accept(*this));
}
auto res = function.function_(arguments.data(), arguments.size(), function_ctx);
MG_ASSERT(res.GetMemoryResource() == ctx_->memory);
return res;
}
}
TypedValue Visit(Reduce &reduce) override {
auto list_value = reduce.list_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (list_value.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("REDUCE expected a list, got {}.", list_value.type());
}
const auto &list = list_value.ValueList();
const auto &element_symbol = symbol_table_->at(*reduce.identifier_);
const auto &accumulator_symbol = symbol_table_->at(*reduce.accumulator_);
auto accumulator = reduce.initializer_->Accept(*this);
for (const auto &element : list) {
frame_->at(accumulator_symbol) = accumulator;
frame_->at(element_symbol) = element;
accumulator = reduce.expression_->Accept(*this);
}
return accumulator;
}
TypedValue Visit(Extract &extract) override {
auto list_value = extract.list_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (list_value.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("EXTRACT expected a list, got {}.", list_value.type());
}
const auto &list = list_value.ValueList();
const auto &element_symbol = symbol_table_->at(*extract.identifier_);
typename TypedValue::TVector result(ctx_->memory);
result.reserve(list.size());
for (const auto &element : list) {
if (element.IsNull()) {
result.emplace_back();
} else {
frame_->at(element_symbol) = element;
result.emplace_back(extract.expression_->Accept(*this));
}
}
return TypedValue(result, ctx_->memory);
}
TypedValue Visit(All &all) override {
auto list_value = all.list_expression_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (list_value.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("ALL expected a list, got {}.", list_value.type());
}
const auto &list = list_value.ValueList();
const auto &symbol = symbol_table_->at(*all.identifier_);
bool has_null_elements = false;
bool has_value = false;
for (const auto &element : list) {
frame_->at(symbol) = element;
auto result = all.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of ALL must evaluate to boolean, got {}.", result.type());
}
if (!result.IsNull()) {
has_value = true;
if (!result.ValueBool()) {
return TypedValue(false, ctx_->memory);
}
} else {
has_null_elements = true;
}
}
if (!has_value) {
return TypedValue(ctx_->memory);
}
if (has_null_elements) {
return TypedValue(false, ctx_->memory);
} else {
return TypedValue(true, ctx_->memory);
}
}
TypedValue Visit(Single &single) override {
auto list_value = single.list_expression_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (list_value.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("SINGLE expected a list, got {}.", list_value.type());
}
const auto &list = list_value.ValueList();
const auto &symbol = symbol_table_->at(*single.identifier_);
bool has_value = false;
bool predicate_satisfied = false;
for (const auto &element : list) {
frame_->at(symbol) = element;
auto result = single.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of SINGLE must evaluate to boolean, got {}.", result.type());
}
if (result.type() == TypedValue::Type::Bool) {
has_value = true;
}
if (result.IsNull() || !result.ValueBool()) {
continue;
}
// Return false if more than one element satisfies the predicate.
if (predicate_satisfied) {
return TypedValue(false, ctx_->memory);
} else {
predicate_satisfied = true;
}
}
if (!has_value) {
return TypedValue(ctx_->memory);
} else {
return TypedValue(predicate_satisfied, ctx_->memory);
}
}
TypedValue Visit(Any &any) override {
auto list_value = any.list_expression_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (list_value.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("ANY expected a list, got {}.", list_value.type());
}
const auto &list = list_value.ValueList();
const auto &symbol = symbol_table_->at(*any.identifier_);
bool has_value = false;
for (const auto &element : list) {
frame_->at(symbol) = element;
auto result = any.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of ANY must evaluate to boolean, got {}.", result.type());
}
if (!result.IsNull()) {
has_value = true;
if (result.ValueBool()) {
return TypedValue(true, ctx_->memory);
}
}
}
// Return Null if all elements are Null
if (!has_value) {
return TypedValue(ctx_->memory);
} else {
return TypedValue(false, ctx_->memory);
}
}
TypedValue Visit(None &none) override {
auto list_value = none.list_expression_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (list_value.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("NONE expected a list, got {}.", list_value.type());
}
const auto &list = list_value.ValueList();
const auto &symbol = symbol_table_->at(*none.identifier_);
bool has_value = false;
for (const auto &element : list) {
frame_->at(symbol) = element;
auto result = none.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of NONE must evaluate to boolean, got {}.", result.type());
}
if (!result.IsNull()) {
has_value = true;
if (result.ValueBool()) {
return TypedValue(false, ctx_->memory);
}
}
}
// Return Null if all elements are Null
if (!has_value) {
return TypedValue(ctx_->memory);
} else {
return TypedValue(true, ctx_->memory);
}
}
TypedValue Visit(ParameterLookup &param_lookup) override {
return TypedValue(conv_(ctx_->parameters.AtTokenPosition(param_lookup.token_position_)), ctx_->memory);
}
TypedValue Visit(RegexMatch &regex_match) override {
auto target_string_value = regex_match.string_expr_->Accept(*this);
auto regex_value = regex_match.regex_->Accept(*this);
if (target_string_value.IsNull() || regex_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (regex_value.type() != TypedValue::Type::String) {
throw ExpressionRuntimeException("Regular expression must evaluate to a string, got {}.", regex_value.type());
}
if (target_string_value.type() != TypedValue::Type::String) {
// Instead of error, we return Null which makes it compatible in case we
// use indexed lookup which filters out any non-string properties.
// Assuming a property lookup is the target_string_value.
return TypedValue(ctx_->memory);
}
const auto &target_string = target_string_value.ValueString();
try {
std::regex regex(regex_value.ValueString());
return TypedValue(std::regex_match(target_string, regex), ctx_->memory);
} catch (const std::regex_error &e) {
throw ExpressionRuntimeException("Regex error in '{}': {}", regex_value.ValueString(), e.what());
}
}
private:
template <class TRecordAccessor, class TTag = Tag,
class TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, TypedValue>>
TReturnType GetProperty(const TRecordAccessor &record_accessor, PropertyIx prop) {
auto maybe_prop = record_accessor.GetProperty(prop.name);
// Handler non existent property
return conv_(maybe_prop);
}
template <class TRecordAccessor, class TTag = Tag,
class TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, TypedValue>>
TReturnType GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) {
auto maybe_prop = record_accessor.GetProperty(std::string(name));
// Handler non existent property
return conv_(maybe_prop);
}
template <class TRecordAccessor, class TTag = Tag,
class TReturnType = std::enable_if_t<std::is_same_v<TTag, StorageTag>, TypedValue>>
TypedValue GetProperty(const TRecordAccessor &record_accessor, PropertyIx prop) {
auto maybe_prop = record_accessor.GetProperty(view_, ctx_->properties[prop.ix]);
if (maybe_prop.HasError() && maybe_prop.GetError() == Error::NONEXISTENT_OBJECT) {
// This is a very nasty and temporary hack in order to make MERGE work.
// The old storage had the following logic when returning an `OLD` view:
// `return old ? old : new`. That means that if the `OLD` view didn't
// exist, it returned the NEW view. With this hack we simulate that
// behavior.
// TODO (mferencevic, teon.banek): Remove once MERGE is reimplemented.
maybe_prop = record_accessor.GetProperty(StorageView::NEW, ctx_->properties[prop.ix]);
}
if (maybe_prop.HasError()) {
switch (maybe_prop.GetError()) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
throw ExpressionRuntimeException("Unexpected error when getting a property.");
}
}
return conv_(*maybe_prop, ctx_->memory);
}
template <class TRecordAccessor, class TTag = Tag,
class TReturnType = std::enable_if_t<std::is_same_v<TTag, StorageTag>, TypedValue>>
TypedValue GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) {
auto maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
if (maybe_prop.HasError() && maybe_prop.GetError() == Error::NONEXISTENT_OBJECT) {
// This is a very nasty and temporary hack in order to make MERGE work.
// The old storage had the following logic when returning an `OLD` view:
// `return old ? old : new`. That means that if the `OLD` view didn't
// exist, it returned the NEW view. With this hack we simulate that
// behavior.
// TODO (mferencevic, teon.banek): Remove once MERGE is reimplemented.
maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
}
if (maybe_prop.HasError()) {
switch (maybe_prop.GetError()) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
throw ExpressionRuntimeException("Unexpected error when getting a property.");
}
}
return conv_(*maybe_prop, ctx_->memory);
}
LabelId GetLabel(LabelIx label) { return ctx_->labels[label.ix]; }
Frame<TypedValue> *frame_;
const SymbolTable *symbol_table_;
const EvaluationContext *ctx_;
DbAccessor *dba_;
// which switching approach should be used when evaluating
StorageView view_;
ConvFunctor conv_;
};
/// A helper function for evaluating an expression that's an int.
///
/// @param what - Name of what's getting evaluated. Used for user feedback (via
/// exception) when the evaluated value is not an int.
/// @throw ExpressionRuntimeException if expression doesn't evaluate to an int.
template <typename ExpressionEvaluator>
int64_t EvaluateInt(ExpressionEvaluator *evaluator, Expression *expr, const std::string &what) {
TypedValue value = expr->Accept(*evaluator);
try {
return value.ValueInt();
} catch (TypedValueException &e) {
throw ExpressionRuntimeException(what + " must be an int");
}
}
template <typename ExpressionEvaluator>
std::optional<size_t> EvaluateMemoryLimit(ExpressionEvaluator *eval, Expression *memory_limit, size_t memory_scale) {
if (!memory_limit) return std::nullopt;
auto limit_value = memory_limit->Accept(*eval);
if (!limit_value.IsInt() || limit_value.ValueInt() <= 0)
throw ExpressionRuntimeException("Memory limit must be a non-negative integer.");
size_t limit = limit_value.ValueInt();
if (std::numeric_limits<size_t>::max() / memory_scale < limit)
throw ExpressionRuntimeException("Memory limit overflow.");
return limit * memory_scale;
}
} // namespace memgraph::expr

View File

@ -0,0 +1,45 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <vector>
#include "expr/semantic/symbol_table.hpp"
#include "utils/logging.hpp"
#include "utils/memory.hpp"
#include "utils/pmr/vector.hpp"
namespace memgraph::expr {
template <typename TypedValue>
class Frame {
public:
/// Create a Frame of given size backed by a utils::NewDeleteResource()
explicit Frame(int64_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
Frame(int64_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
TypedValue &operator[](const Symbol &symbol) { return elems_[symbol.position()]; }
const TypedValue &operator[](const Symbol &symbol) const { return elems_[symbol.position()]; }
TypedValue &at(const Symbol &symbol) { return elems_.at(symbol.position()); }
const TypedValue &at(const Symbol &symbol) const { return elems_.at(symbol.position()); }
auto &elems() { return elems_; }
utils::MemoryResource *GetMemoryResource() const { return elems_.get_allocator().GetMemoryResource(); }
private:
utils::pmr::vector<TypedValue> elems_;
};
} // namespace memgraph::expr

184
src/expr/parsing.cpp Normal file
View File

@ -0,0 +1,184 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "expr/parsing.hpp"
#include <cctype>
#include <codecvt>
#include <locale>
#include <stdexcept>
#include "expr/exceptions.hpp"
#include "utils/logging.hpp"
#include "utils/string.hpp"
namespace memgraph::expr {
int64_t ParseIntegerLiteral(const std::string &s) {
try {
// Not really correct since long long can have a bigger range than int64_t.
return static_cast<int64_t>(std::stoll(s, 0, 0));
} catch (const std::out_of_range &) {
throw SemanticException("Integer literal exceeds 64 bits.");
}
}
std::string ParseStringLiteral(const std::string &s) {
// These functions is declared as lambda since its semantics is highly
// specific for this conxtext and shouldn't be used elsewhere.
auto EncodeEscapedUnicodeCodepointUtf32 = [](const std::string &s, int &i) {
const int kLongUnicodeLength = 8;
int j = i + 1;
while (j < static_cast<int>(s.size()) - 1 && j < i + kLongUnicodeLength + 1 && isxdigit(s[j])) {
++j;
}
if (j - i == kLongUnicodeLength + 1) {
char32_t t = stoi(s.substr(i + 1, kLongUnicodeLength), 0, 16);
i += kLongUnicodeLength;
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> converter;
return converter.to_bytes(t);
}
throw SyntaxException(
"Expected 8 hex digits as unicode codepoint started with \\U. "
"Use \\u for 4 hex digits format.");
};
auto EncodeEscapedUnicodeCodepointUtf16 = [](const std::string &s, int &i) {
const int kShortUnicodeLength = 4;
int j = i + 1;
while (j < static_cast<int>(s.size()) - 1 && j < i + kShortUnicodeLength + 1 && isxdigit(s[j])) {
++j;
}
if (j - i >= kShortUnicodeLength + 1) {
char16_t t = stoi(s.substr(i + 1, kShortUnicodeLength), 0, 16);
if (t >= 0xD800 && t <= 0xDBFF) {
// t is high surrogate pair. Expect one more utf16 codepoint.
j = i + kShortUnicodeLength + 1;
if (j >= static_cast<int>(s.size()) - 1 || s[j] != '\\') {
throw SemanticException("Invalid UTF codepoint.");
}
++j;
if (j >= static_cast<int>(s.size()) - 1 || (s[j] != 'u' && s[j] != 'U')) {
throw SemanticException("Invalid UTF codepoint.");
}
++j;
int k = j;
while (k < static_cast<int>(s.size()) - 1 && k < j + kShortUnicodeLength && isxdigit(s[k])) {
++k;
}
if (k != j + kShortUnicodeLength) {
throw SemanticException("Invalid UTF codepoint.");
}
char16_t surrogates[3] = {t, static_cast<char16_t>(stoi(s.substr(j, kShortUnicodeLength), 0, 16)), 0};
i += kShortUnicodeLength + 2 + kShortUnicodeLength;
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
return converter.to_bytes(surrogates);
} else {
i += kShortUnicodeLength;
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
return converter.to_bytes(t);
}
}
throw SyntaxException(
"Expected 4 hex digits as unicode codepoint started with \\u. "
"Use \\U for 8 hex digits format.");
};
std::string unescaped;
bool escape = false;
// First and last char is quote, we don't need to look at them.
for (int i = 1; i < static_cast<int>(s.size()) - 1; ++i) {
if (escape) {
switch (s[i]) {
case '\\':
unescaped += '\\';
break;
case '\'':
unescaped += '\'';
break;
case '"':
unescaped += '"';
break;
case 'B':
case 'b':
unescaped += '\b';
break;
case 'F':
case 'f':
unescaped += '\f';
break;
case 'N':
case 'n':
unescaped += '\n';
break;
case 'R':
case 'r':
unescaped += '\r';
break;
case 'T':
case 't':
unescaped += '\t';
break;
case 'U':
try {
unescaped += EncodeEscapedUnicodeCodepointUtf32(s, i);
} catch (const std::range_error &) {
throw SemanticException("Invalid UTF codepoint.");
}
break;
case 'u':
try {
unescaped += EncodeEscapedUnicodeCodepointUtf16(s, i);
} catch (const std::range_error &) {
throw SemanticException("Invalid UTF codepoint.");
}
break;
default:
// This should never happen, except grammar changes and we don't
// notice change in this production.
DLOG_FATAL("can't happen");
throw std::exception();
}
escape = false;
} else if (s[i] == '\\') {
escape = true;
} else {
unescaped += s[i];
}
}
return unescaped;
}
double ParseDoubleLiteral(const std::string &s) {
try {
return utils::ParseDouble(s);
} catch (const utils::BasicException &) {
throw SemanticException("Couldn't parse string to double.");
}
}
std::string ParseParameter(const std::string &s) {
DMG_ASSERT(s[0] == '$', "Invalid string passed as parameter name");
if (s[1] != '`') return s.substr(1);
// If parameter name is escaped symbolic name then symbolic name should be
// unescaped and leading and trailing backquote should be removed.
DMG_ASSERT(s.size() > 3U && s.back() == '`', "Invalid string passed as parameter name");
std::string out;
for (int i = 2; i < static_cast<int>(s.size()) - 1; ++i) {
if (s[i] == '`') {
++i;
}
out.push_back(s[i]);
}
return out;
}
} // namespace memgraph::expr

27
src/expr/parsing.hpp Normal file
View File

@ -0,0 +1,27 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file
#pragma once
#include <cstdint>
#include <string>
namespace memgraph::expr {
// These are the functions for parsing literals and parameter names from
// opencypher query.
int64_t ParseIntegerLiteral(const std::string &s);
std::string ParseStringLiteral(const std::string &s);
double ParseDoubleLiteral(const std::string &s);
std::string ParseParameter(const std::string &s);
} // namespace memgraph::expr

View File

@ -0,0 +1,87 @@
;; Copyright 2022 Memgraph Ltd.
;;
;; Use of this software is governed by the Business Source License
;; included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
;; License, and you may not use this file except in compliance with the Business Source License.
;;
;; As of the Change Date specified in that file, in accordance with
;; the Business Source License, use of this software will be governed
;; by the Apache License, Version 2.0, included in the file
;; licenses/APL.txt.
#>cpp
#pragma once
#include <string>
#include "utils/typeinfo.hpp"
cpp<#
(lcp:namespace memgraph)
(lcp:namespace expr)
(lcp:define-class symbol ()
((name "std::string" :scope :public)
(position :int64_t :scope :public)
(user-declared :bool :initval "true" :scope :public)
(type "Type" :initval "Type::ANY" :scope :public)
(token-position :int64_t :initval "-1" :scope :public))
(:public
;; This is similar to TypedValue::Type, but this has `Any` type.
;; TODO: Make a better Type structure which can store a generic List.
(lcp:define-enum type (any vertex edge path number edge-list)
(:serialize))
#>cpp
// TODO: Generate enum to string conversion from LCP. Note, that this is
// displayed to the end user, so we may want to have a pretty name of each
// value.
static std::string TypeToString(Type type) {
const char *enum_string[] = {"Any", "Vertex", "Edge",
"Path", "Number", "EdgeList"};
return enum_string[static_cast<int>(type)];
}
Symbol() {}
Symbol(const std::string &name, int position, bool user_declared,
Type type = Type::ANY, int token_position = -1)
: name_(name),
position_(position),
user_declared_(user_declared),
type_(type),
token_position_(token_position) {}
bool operator==(const Symbol &other) const {
return position_ == other.position_ && name_ == other.name_ &&
type_ == other.type_;
}
bool operator!=(const Symbol &other) const { return !operator==(other); }
// TODO: Remove these since members are public
const auto &name() const { return name_; }
int position() const { return position_; }
Type type() const { return type_; }
bool user_declared() const { return user_declared_; }
int token_position() const { return token_position_; }
cpp<#)
(:serialize (:slk)))
(lcp:pop-namespace) ;; expr
(lcp:pop-namespace) ;; memgraph
#>cpp
namespace std {
template <>
struct hash<memgraph::expr::Symbol> {
size_t operator()(const memgraph::expr::Symbol &symbol) const {
size_t prime = 265443599u;
size_t hash = std::hash<int>{}(symbol.position());
hash ^= prime * std::hash<std::string>{}(symbol.name());
hash ^= prime * std::hash<int>{}(static_cast<int>(symbol.type()));
return hash;
}
};
} // namespace std
cpp<#

View File

@ -0,0 +1,712 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
// Copyright 2017 Memgraph
//
// Created by Teon Banek on 11-03-2017
#pragma once
#include <algorithm>
#include <optional>
#include <ranges>
#include <unordered_map>
#include <unordered_set>
#include <variant>
#include <vector>
#include "expr/ast.hpp"
#include "expr/ast/ast_visitor.hpp"
#include "expr/exceptions.hpp"
#include "expr/semantic/symbol_table.hpp"
namespace memgraph::expr {
namespace detail {
inline std::unordered_map<std::string, Identifier *> GeneratePredefinedIdentifierMap(
const std::vector<Identifier *> &predefined_identifiers) {
std::unordered_map<std::string, Identifier *> identifier_map;
for (const auto &identifier : predefined_identifiers) {
identifier_map.emplace(identifier->name_, identifier);
}
return identifier_map;
}
} // namespace detail
/// Visits the AST and generates symbols for variables.
///
/// During the process of symbol generation, simple semantic checks are
/// performed. Such as, redeclaring a variable or conflicting expectations of
/// variable types.
class SymbolGenerator : public HierarchicalTreeVisitor {
public:
explicit SymbolGenerator(SymbolTable *symbol_table, const std::vector<Identifier *> &predefined_identifiers)
: symbol_table_(symbol_table),
predefined_identifiers_{detail::GeneratePredefinedIdentifierMap(predefined_identifiers)},
scopes_(1, Scope()) {}
using HierarchicalTreeVisitor::PostVisit;
using HierarchicalTreeVisitor::PreVisit;
using HierarchicalTreeVisitor::Visit;
using typename HierarchicalTreeVisitor::ReturnType;
// Query
bool PreVisit(SingleQuery & /*unused*/) override {
prev_return_names_ = curr_return_names_;
curr_return_names_.clear();
return true;
}
// Union
bool PreVisit(CypherUnion & /*unused*/) override {
scopes_.back() = Scope();
return true;
}
bool PostVisit(CypherUnion &cypher_union) override {
if (prev_return_names_ != curr_return_names_) {
throw SemanticException("All subqueries in an UNION must have the same column names.");
}
// create new symbols for the result of the union
for (const auto &name : curr_return_names_) {
auto symbol = CreateSymbol(name, false);
cypher_union.union_symbols_.push_back(symbol);
}
return true;
}
// Clauses
bool PreVisit(Create & /*unused*/) override {
scopes_.back().in_create = true;
return true;
}
bool PostVisit(Create & /*unused*/) override {
scopes_.back().in_create = false;
return true;
}
bool PreVisit(CallProcedure &call_proc) override {
for (auto *expr : call_proc.arguments_) {
expr->Accept(*this);
}
return false;
}
bool PostVisit(CallProcedure &call_proc) override {
for (auto *ident : call_proc.result_identifiers_) {
if (HasSymbolLocalScope(ident->name_)) {
throw RedeclareVariableError(ident->name_);
}
ident->MapTo(CreateSymbol(ident->name_, true));
}
return true;
}
bool PreVisit(LoadCsv & /*unused*/) override { return false; }
bool PostVisit(LoadCsv &load_csv) override {
if (HasSymbolLocalScope(load_csv.row_var_->name_)) {
throw RedeclareVariableError(load_csv.row_var_->name_);
}
load_csv.row_var_->MapTo(CreateSymbol(load_csv.row_var_->name_, true));
return true;
}
bool PreVisit(Return &ret) override {
auto &scope = scopes_.back();
scope.in_return = true;
VisitReturnBody(ret.body_);
scope.in_return = false;
return false; // We handled the traversal ourselves.
}
bool PostVisit(Return & /*unused*/) override {
for (const auto &name_symbol : scopes_.back().symbols) curr_return_names_.insert(name_symbol.first);
return true;
}
bool PreVisit(With &with) override {
auto &scope = scopes_.back();
scope.in_with = true;
VisitReturnBody(with.body_, with.where_);
scope.in_with = false;
return false; // We handled the traversal ourselves.
}
bool PreVisit(Where & /*unused*/) override {
scopes_.back().in_where = true;
return true;
}
bool PostVisit(Where & /*unused*/) override {
scopes_.back().in_where = false;
return true;
}
bool PreVisit(Merge & /*unused*/) override {
scopes_.back().in_merge = true;
return true;
}
bool PostVisit(Merge & /*unused*/) override {
scopes_.back().in_merge = false;
return true;
}
bool PostVisit(Unwind &unwind) override {
const auto &name = unwind.named_expression_->name_;
if (HasSymbolLocalScope(name)) {
throw RedeclareVariableError(name);
}
unwind.named_expression_->MapTo(CreateSymbol(name, true));
return true;
}
bool PreVisit(Match & /*unused*/) override {
scopes_.back().in_match = true;
return true;
}
bool PostVisit(Match & /*unused*/) override {
auto &scope = scopes_.back();
scope.in_match = false;
// Check variables in property maps after visiting Match, so that they can
// reference symbols out of bind order.
for (auto &ident : scope.identifiers_in_match) {
if (!HasSymbolLocalScope(ident->name_) && !ConsumePredefinedIdentifier(ident->name_))
throw UnboundVariableError(ident->name_);
ident->MapTo(scope.symbols[ident->name_]);
}
scope.identifiers_in_match.clear();
return true;
}
bool PreVisit(Foreach &for_each) override {
const auto &name = for_each.named_expression_->name_;
scopes_.emplace_back(Scope());
scopes_.back().in_foreach = true;
for_each.named_expression_->MapTo(
CreateSymbol(name, true, Symbol::Type::ANY, for_each.named_expression_->token_position_));
return true;
}
bool PostVisit(Foreach & /*unused*/) override {
scopes_.pop_back();
return true;
}
// Expressions
ReturnType Visit(Identifier &ident) override {
auto &scope = scopes_.back();
if (scope.in_skip || scope.in_limit) {
throw SemanticException("Variables are not allowed in {}.", scope.in_skip ? "SKIP" : "LIMIT");
}
Symbol symbol;
if (scope.in_pattern && !(scope.in_node_atom || scope.visiting_edge)) {
// If we are in the pattern, and outside of a node or an edge, the
// identifier is the pattern name.
symbol = GetOrCreateSymbolLocalScope(ident.name_, ident.user_declared_, Symbol::Type::PATH);
} else if (scope.in_pattern && scope.in_pattern_atom_identifier) {
// Patterns used to create nodes and edges cannot redeclare already
// established bindings. Declaration only happens in single node
// patterns and in edge patterns. OpenCypher example,
// `MATCH (n) CREATE (n)` should throw an error that `n` is already
// declared. While `MATCH (n) CREATE (n) -[:R]-> (n)` is allowed,
// since `n` now references the bound node instead of declaring it.
if ((scope.in_create_node || scope.in_create_edge) && HasSymbolLocalScope(ident.name_)) {
throw RedeclareVariableError(ident.name_);
}
auto type = Symbol::Type::VERTEX;
if (scope.visiting_edge) {
// Edge referencing is not allowed (like in Neo4j):
// `MATCH (n) - [r] -> (n) - [r] -> (n) RETURN r` is not allowed.
if (HasSymbolLocalScope(ident.name_)) {
throw RedeclareVariableError(ident.name_);
}
type = scope.visiting_edge->IsVariable() ? Symbol::Type::EDGE_LIST : Symbol::Type::EDGE;
}
symbol = GetOrCreateSymbolLocalScope(ident.name_, ident.user_declared_, type);
} else if (scope.in_pattern && !scope.in_pattern_atom_identifier && scope.in_match) {
if (scope.in_edge_range && scope.visiting_edge->identifier_->name_ == ident.name_) {
// Prevent variable path bounds to reference the identifier which is bound
// by the variable path itself.
throw UnboundVariableError(ident.name_);
}
// Variables in property maps or bounds of variable length path during MATCH
// can reference symbols bound later in the same MATCH. We collect them
// here, so that they can be checked after visiting Match.
scope.identifiers_in_match.emplace_back(&ident);
} else {
// Everything else references a bound symbol.
if (!HasSymbol(ident.name_) && !ConsumePredefinedIdentifier(ident.name_)) throw UnboundVariableError(ident.name_);
symbol = GetOrCreateSymbol(ident.name_, ident.user_declared_, Symbol::Type::ANY);
}
ident.MapTo(symbol);
return true;
}
ReturnType Visit(PrimitiveLiteral & /*unused*/) override { return true; }
ReturnType Visit(ParameterLookup & /*unused*/) override { return true; }
bool PreVisit(Aggregation &aggr) override {
auto &scope = scopes_.back();
// Check if the aggregation can be used in this context. This check should
// probably move to a separate phase, which checks if the query is well
// formed.
if ((!scope.in_return && !scope.in_with) || scope.in_order_by || scope.in_skip || scope.in_limit ||
scope.in_where) {
throw SemanticException("Aggregation functions are only allowed in WITH and RETURN.");
}
if (scope.in_aggregation) {
throw SemanticException(
"Using aggregation functions inside aggregation functions is not "
"allowed.");
}
if (scope.num_if_operators) {
// Neo allows aggregations here and produces very interesting behaviors.
// To simplify implementation at this moment we decided to completely
// disallow aggregations inside of the CASE.
// However, in some cases aggregation makes perfect sense, for example:
// CASE count(n) WHEN 10 THEN "YES" ELSE "NO" END.
// TODO: Rethink of allowing aggregations in some parts of the CASE
// construct.
throw SemanticException("Using aggregation functions inside of CASE is not allowed.");
}
// Create a virtual symbol for aggregation result.
// Currently, we only have aggregation operators which return numbers.
auto aggr_name = Aggregation::OpToString(aggr.op_) + std::to_string(aggr.symbol_pos_);
aggr.MapTo(CreateSymbol(aggr_name, false, Symbol::Type::NUMBER));
scope.in_aggregation = true;
scope.has_aggregation = true;
return true;
}
bool PostVisit(Aggregation & /*unused*/) override {
scopes_.back().in_aggregation = false;
return true;
}
bool PreVisit(IfOperator & /*unused*/) override {
++scopes_.back().num_if_operators;
return true;
}
bool PostVisit(IfOperator & /*unused*/) override {
--scopes_.back().num_if_operators;
return true;
}
bool PreVisit(All &all) override {
all.list_expression_->Accept(*this);
VisitWithIdentifiers(all.where_->expression_, {all.identifier_});
return false;
}
bool PreVisit(Single &single) override {
single.list_expression_->Accept(*this);
VisitWithIdentifiers(single.where_->expression_, {single.identifier_});
return false;
}
bool PreVisit(Any &any) override {
any.list_expression_->Accept(*this);
VisitWithIdentifiers(any.where_->expression_, {any.identifier_});
return false;
}
bool PreVisit(None &none) override {
none.list_expression_->Accept(*this);
VisitWithIdentifiers(none.where_->expression_, {none.identifier_});
return false;
}
bool PreVisit(Reduce &reduce) override {
reduce.initializer_->Accept(*this);
reduce.list_->Accept(*this);
VisitWithIdentifiers(reduce.expression_, {reduce.accumulator_, reduce.identifier_});
return false;
}
bool PreVisit(Extract &extract) override {
extract.list_->Accept(*this);
VisitWithIdentifiers(extract.expression_, {extract.identifier_});
return false;
}
// Pattern and its subparts.
bool PreVisit(Pattern &pattern) override {
auto &scope = scopes_.back();
scope.in_pattern = true;
if ((scope.in_create || scope.in_merge) && pattern.atoms_.size() == 1U) {
MG_ASSERT(utils::IsSubtype(*pattern.atoms_[0], NodeAtom::kType), "Expected a single NodeAtom in Pattern");
scope.in_create_node = true;
}
return true;
}
bool PostVisit(Pattern & /*unused*/) override {
auto &scope = scopes_.back();
scope.in_pattern = false;
scope.in_create_node = false;
return true;
}
bool PreVisit(NodeAtom &node_atom) override {
auto &scope = scopes_.back();
auto check_node_semantic = [&node_atom, &scope, this](const bool props_or_labels) {
const auto &node_name = node_atom.identifier_->name_;
if ((scope.in_create || scope.in_merge) && props_or_labels && HasSymbolLocalScope(node_name)) {
throw SemanticException("Cannot create node '" + node_name +
"' with labels or properties, because it is already declared.");
}
scope.in_pattern_atom_identifier = true;
node_atom.identifier_->Accept(*this);
scope.in_pattern_atom_identifier = false;
};
scope.in_node_atom = true;
if (auto *properties = std::get_if<std::unordered_map<PropertyIx, Expression *>>(&node_atom.properties_)) {
bool props_or_labels = !properties->empty() || !node_atom.labels_.empty();
check_node_semantic(props_or_labels);
for (auto kv : *properties) {
kv.second->Accept(*this);
}
return false;
}
auto &properties_parameter = std::get<ParameterLookup *>(node_atom.properties_);
bool props_or_labels = !properties_parameter || !node_atom.labels_.empty();
check_node_semantic(props_or_labels);
properties_parameter->Accept(*this);
return false;
}
bool PostVisit(NodeAtom & /*unused*/) override {
scopes_.back().in_node_atom = false;
return true;
}
bool PreVisit(EdgeAtom &edge_atom) override {
auto &scope = scopes_.back();
scope.visiting_edge = &edge_atom;
if (scope.in_create || scope.in_merge) {
scope.in_create_edge = true;
if (edge_atom.edge_types_.size() != 1U) {
throw SemanticException(
"A single relationship type must be specified "
"when creating an edge.");
}
if (scope.in_create && // Merge allows bidirectionality
edge_atom.direction_ == EdgeAtom::Direction::BOTH) {
throw SemanticException(
"Bidirectional relationship are not supported "
"when creating an edge");
}
if (edge_atom.IsVariable()) {
throw SemanticException(
"Variable length relationships are not supported when creating an "
"edge.");
}
}
if (auto *properties = std::get_if<std::unordered_map<PropertyIx, Expression *>>(&edge_atom.properties_)) {
for (auto kv : *properties) {
kv.second->Accept(*this);
}
} else {
std::get<ParameterLookup *>(edge_atom.properties_)->Accept(*this);
}
if (edge_atom.IsVariable()) {
scope.in_edge_range = true;
if (edge_atom.lower_bound_) {
edge_atom.lower_bound_->Accept(*this);
}
if (edge_atom.upper_bound_) {
edge_atom.upper_bound_->Accept(*this);
}
scope.in_edge_range = false;
scope.in_pattern = false;
if (edge_atom.filter_lambda_.expression) {
VisitWithIdentifiers(edge_atom.filter_lambda_.expression,
{edge_atom.filter_lambda_.inner_edge, edge_atom.filter_lambda_.inner_node});
} else {
// Create inner symbols, but don't bind them in scope, since they are to
// be used in the missing filter expression.
auto *inner_edge = edge_atom.filter_lambda_.inner_edge;
inner_edge->MapTo(
symbol_table_->CreateSymbol(inner_edge->name_, inner_edge->user_declared_, Symbol::Type::EDGE));
auto *inner_node = edge_atom.filter_lambda_.inner_node;
inner_node->MapTo(
symbol_table_->CreateSymbol(inner_node->name_, inner_node->user_declared_, Symbol::Type::VERTEX));
}
if (edge_atom.weight_lambda_.expression) {
VisitWithIdentifiers(edge_atom.weight_lambda_.expression,
{edge_atom.weight_lambda_.inner_edge, edge_atom.weight_lambda_.inner_node});
}
scope.in_pattern = true;
}
scope.in_pattern_atom_identifier = true;
edge_atom.identifier_->Accept(*this);
scope.in_pattern_atom_identifier = false;
if (edge_atom.total_weight_) {
if (HasSymbolLocalScope(edge_atom.total_weight_->name_)) {
throw RedeclareVariableError(edge_atom.total_weight_->name_);
}
edge_atom.total_weight_->MapTo(GetOrCreateSymbolLocalScope(
edge_atom.total_weight_->name_, edge_atom.total_weight_->user_declared_, Symbol::Type::NUMBER));
}
return false;
}
bool PostVisit(EdgeAtom & /*unused*/) override {
auto &scope = scopes_.back();
scope.visiting_edge = nullptr;
scope.in_create_edge = false;
return true;
}
private:
// Scope stores the state of where we are when visiting the AST and a map of
// names to symbols.
struct Scope {
bool in_pattern{false};
bool in_merge{false};
bool in_create{false};
// in_create_node is true if we are creating or merging *only* a node.
// Therefore, it is *not* equivalent to (in_create || in_merge) &&
// in_node_atom.
bool in_create_node{false};
// True if creating an edge;
// shortcut for (in_create || in_merge) && visiting_edge.
bool in_create_edge{false};
bool in_node_atom{false};
EdgeAtom *visiting_edge{nullptr};
bool in_aggregation{false};
bool in_return{false};
bool in_with{false};
bool in_skip{false};
bool in_limit{false};
bool in_order_by{false};
bool in_where{false};
bool in_match{false};
bool in_foreach{false};
// True when visiting a pattern atom (node or edge) identifier, which can be
// reused or created in the pattern itself.
bool in_pattern_atom_identifier{false};
// True when visiting range bounds of a variable path.
bool in_edge_range{false};
// True if the return/with contains an aggregation in any named expression.
bool has_aggregation{false};
// Map from variable names to symbols.
std::map<std::string, Symbol> symbols;
// Identifiers found in property maps of patterns or as variable length path
// bounds in a single Match clause. They need to be checked after visiting
// Match. Identifiers created by naming vertices, edges and paths are *not*
// stored in here.
std::vector<Identifier *> identifiers_in_match;
// Number of nested IfOperators.
int num_if_operators{0};
};
inline static std::optional<Symbol> FindSymbolInScope(const std::string &name, const Scope &scope,
Symbol::Type type) {
if (auto it = scope.symbols.find(name); it != scope.symbols.end()) {
const auto &symbol = it->second;
// Unless we have `ANY` type, check that types match.
if (type != Symbol::Type::ANY && symbol.type() != Symbol::Type::ANY && type != symbol.type()) {
throw TypeMismatchError(name, Symbol::TypeToString(symbol.type()), Symbol::TypeToString(type));
}
return symbol;
}
return std::nullopt;
}
bool HasSymbol(const std::string &name) const {
return std::ranges::any_of(scopes_, [&name](const auto &scope) { return scope.symbols.contains(name); });
}
bool HasSymbolLocalScope(const std::string &name) const { return scopes_.back().symbols.contains(name); }
// @return true if it added a predefined identifier with that name
bool ConsumePredefinedIdentifier(const std::string &name) {
auto it = predefined_identifiers_.find(name);
if (it == predefined_identifiers_.end()) {
return false;
}
// we can only use the predefined identifier in a single scope so we remove it after creating
// a symbol for it
auto &identifier = it->second;
MG_ASSERT(!identifier->user_declared_, "Predefined symbols cannot be user declared!");
identifier->MapTo(CreateSymbol(identifier->name_, identifier->user_declared_));
predefined_identifiers_.erase(it);
return true;
}
// Returns a freshly generated symbol. Previous mapping of the same name to a
// different symbol is replaced with the new one.
Symbol CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY,
int token_position = -1) {
auto symbol = symbol_table_->CreateSymbol(name, user_declared, type, token_position);
scopes_.back().symbols[name] = symbol;
return symbol;
}
Symbol GetOrCreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY) {
// NOLINTNEXTLINE
for (auto scope = scopes_.rbegin(); scope != scopes_.rend(); ++scope) {
if (auto maybe_symbol = FindSymbolInScope(name, *scope, type); maybe_symbol) {
return *maybe_symbol;
}
}
return CreateSymbol(name, user_declared, type);
}
// Returns the symbol by name. If the mapping already exists, checks if the
// types match. Otherwise, returns a new symbol.
Symbol GetOrCreateSymbolLocalScope(const std::string &name, bool user_declared,
Symbol::Type type = Symbol::Type::ANY) {
auto &scope = scopes_.back();
if (auto maybe_symbol = FindSymbolInScope(name, scope, type); maybe_symbol) {
return *maybe_symbol;
}
return CreateSymbol(name, user_declared, type);
}
void VisitReturnBody(ReturnBody &body, Where *where = nullptr) {
auto &scope = scopes_.back();
for (auto &expr : body.named_expressions) {
expr->Accept(*this);
}
std::vector<Symbol> user_symbols;
if (body.all_identifiers) {
// Carry over user symbols because '*' appeared.
for (const auto &sym_pair : scope.symbols) {
if (!sym_pair.second.user_declared()) {
continue;
}
user_symbols.emplace_back(sym_pair.second);
}
if (user_symbols.empty()) {
throw SemanticException("There are no variables in scope to use for '*'.");
}
}
// WITH/RETURN clause removes declarations of all the previous variables and
// declares only those established through named expressions. New declarations
// must not be visible inside named expressions themselves.
bool removed_old_names = false;
if ((!where && body.order_by.empty()) || scope.has_aggregation) {
// WHERE and ORDER BY need to see both the old and new symbols, unless we
// have an aggregation. Therefore, we can clear the symbols immediately if
// there is neither ORDER BY nor WHERE, or we have an aggregation.
scope.symbols.clear();
removed_old_names = true;
}
// Create symbols for named expressions.
std::unordered_set<std::string> new_names;
for (const auto &user_sym : user_symbols) {
new_names.insert(user_sym.name());
scope.symbols[user_sym.name()] = user_sym;
}
for (auto &named_expr : body.named_expressions) {
const auto &name = named_expr->name_;
if (!new_names.insert(name).second) {
throw SemanticException("Multiple results with the same name '{}' are not allowed.", name);
}
// An improvement would be to infer the type of the expression, so that the
// new symbol would have a more specific type.
named_expr->MapTo(CreateSymbol(name, true, Symbol::Type::ANY, named_expr->token_position_));
}
scope.in_order_by = true;
for (const auto &order_pair : body.order_by) {
order_pair.expression->Accept(*this);
}
scope.in_order_by = false;
if (body.skip) {
scope.in_skip = true;
body.skip->Accept(*this);
scope.in_skip = false;
}
if (body.limit) {
scope.in_limit = true;
body.limit->Accept(*this);
scope.in_limit = false;
}
if (where) where->Accept(*this);
if (!removed_old_names) {
// We have an ORDER BY or WHERE, but no aggregation, which means we didn't
// clear the old symbols, so do it now. We cannot just call clear, because
// we've added new symbols.
for (auto sym_it = scope.symbols.begin(); sym_it != scope.symbols.end();) {
if (new_names.find(sym_it->first) == new_names.end()) {
sym_it = scope.symbols.erase(sym_it);
} else {
sym_it++;
}
}
}
scopes_.back().has_aggregation = false;
}
void VisitWithIdentifiers(Expression *expr, const std::vector<Identifier *> &identifiers) {
auto &scope = scopes_.back();
std::vector<std::pair<std::optional<Symbol>, Identifier *>> prev_symbols;
// Collect previous symbols if they exist.
for (const auto &identifier : identifiers) {
std::optional<Symbol> prev_symbol;
auto prev_symbol_it = scope.symbols.find(identifier->name_);
if (prev_symbol_it != scope.symbols.end()) {
prev_symbol = prev_symbol_it->second;
}
identifier->MapTo(CreateSymbol(identifier->name_, identifier->user_declared_));
prev_symbols.emplace_back(prev_symbol, identifier);
}
// Visit the expression with the new symbols bound.
expr->Accept(*this);
// Restore back to previous symbols.
for (const auto &prev : prev_symbols) {
const auto &prev_symbol = prev.first;
const auto &identifier = prev.second;
if (prev_symbol) {
scope.symbols[identifier->name_] = *prev_symbol;
} else {
scope.symbols.erase(identifier->name_);
}
}
}
SymbolTable *symbol_table_;
// Identifiers which are injected from outside the query. Each identifier
// is mapped by its name.
std::unordered_map<std::string, Identifier *> predefined_identifiers_;
std::vector<Scope> scopes_;
std::unordered_set<std::string> prev_return_names_;
std::unordered_set<std::string> curr_return_names_;
};
inline SymbolTable MakeSymbolTable(CypherQuery *query, const std::vector<Identifier *> &predefined_identifiers = {}) {
SymbolTable symbol_table;
SymbolGenerator symbol_generator(&symbol_table, predefined_identifiers);
query->single_query_->Accept(symbol_generator);
for (auto *cypher_union : query->cypher_unions_) {
cypher_union->Accept(symbol_generator);
}
return symbol_table;
}
} // namespace memgraph::expr

View File

@ -0,0 +1,64 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <map>
#include <string>
#include "expr/ast.hpp"
#include "expr/semantic/symbol.hpp"
#include "utils/logging.hpp"
namespace memgraph::expr {
class SymbolTable final {
public:
SymbolTable() {}
const Symbol &CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY,
int32_t token_position = -1) {
MG_ASSERT(table_.size() <= std::numeric_limits<int32_t>::max(),
"SymbolTable size doesn't fit into 32-bit integer!");
auto got = table_.emplace(position_, Symbol(name, position_, user_declared, type, token_position));
MG_ASSERT(got.second, "Duplicate symbol ID!");
position_++;
return got.first->second;
}
// TODO(buda): This is the same logic as in the cypher_main_visitor. During
// parsing phase symbol table doesn't exist. Figure out a better solution.
const Symbol &CreateAnonymousSymbol(Symbol::Type type = Symbol::Type::ANY) {
int id = 1;
while (true) {
static const std::string &kAnonPrefix = "anon";
std::string name_candidate = kAnonPrefix + std::to_string(id++);
if (std::find_if(std::begin(table_), std::end(table_), [&name_candidate](const auto &item) -> bool {
return item.second.name_ == name_candidate;
}) == std::end(table_)) {
return CreateSymbol(name_candidate, false, type);
}
}
}
const Symbol &at(const Identifier &ident) const { return table_.at(ident.symbol_pos_); }
const Symbol &at(const NamedExpression &nexpr) const { return table_.at(nexpr.symbol_pos_); }
const Symbol &at(const Aggregation &aggr) const { return table_.at(aggr.symbol_pos_); }
// TODO: Remove these since members are public
int32_t max_position() const { return static_cast<int32_t>(table_.size()); }
const auto &table() const { return table_; }
int32_t position_{0};
std::map<int32_t, Symbol> table_;
};
} // namespace memgraph::expr

1514
src/expr/typed_value.hpp Normal file

File diff suppressed because it is too large Load Diff

64
src/glue/v2/auth.cpp Normal file
View File

@ -0,0 +1,64 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/v2/auth.hpp"
namespace memgraph::glue::v2 {
auth::Permission PrivilegeToPermission(query::v2::AuthQuery::Privilege privilege) {
switch (privilege) {
case query::v2::AuthQuery::Privilege::MATCH:
return auth::Permission::MATCH;
case query::v2::AuthQuery::Privilege::CREATE:
return auth::Permission::CREATE;
case query::v2::AuthQuery::Privilege::MERGE:
return auth::Permission::MERGE;
case query::v2::AuthQuery::Privilege::DELETE:
return auth::Permission::DELETE;
case query::v2::AuthQuery::Privilege::SET:
return auth::Permission::SET;
case query::v2::AuthQuery::Privilege::REMOVE:
return auth::Permission::REMOVE;
case query::v2::AuthQuery::Privilege::INDEX:
return auth::Permission::INDEX;
case query::v2::AuthQuery::Privilege::STATS:
return auth::Permission::STATS;
case query::v2::AuthQuery::Privilege::CONSTRAINT:
return auth::Permission::CONSTRAINT;
case query::v2::AuthQuery::Privilege::DUMP:
return auth::Permission::DUMP;
case query::v2::AuthQuery::Privilege::REPLICATION:
return auth::Permission::REPLICATION;
case query::v2::AuthQuery::Privilege::DURABILITY:
return auth::Permission::DURABILITY;
case query::v2::AuthQuery::Privilege::READ_FILE:
return auth::Permission::READ_FILE;
case query::v2::AuthQuery::Privilege::FREE_MEMORY:
return auth::Permission::FREE_MEMORY;
case query::v2::AuthQuery::Privilege::TRIGGER:
return auth::Permission::TRIGGER;
case query::v2::AuthQuery::Privilege::CONFIG:
return auth::Permission::CONFIG;
case query::v2::AuthQuery::Privilege::AUTH:
return auth::Permission::AUTH;
case query::v2::AuthQuery::Privilege::STREAM:
return auth::Permission::STREAM;
case query::v2::AuthQuery::Privilege::MODULE_READ:
return auth::Permission::MODULE_READ;
case query::v2::AuthQuery::Privilege::MODULE_WRITE:
return auth::Permission::MODULE_WRITE;
case query::v2::AuthQuery::Privilege::WEBSOCKET:
return auth::Permission::WEBSOCKET;
case query::v2::AuthQuery::Privilege::SCHEMA:
return auth::Permission::SCHEMA;
}
}
} // namespace memgraph::glue::v2

23
src/glue/v2/auth.hpp Normal file
View File

@ -0,0 +1,23 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "auth/models.hpp"
#include "query/v2/frontend/ast/ast.hpp"
namespace memgraph::glue::v2 {
/**
* This function converts query::AuthQuery::Privilege to its corresponding
* auth::Permission.
*/
auth::Permission PrivilegeToPermission(query::v2::AuthQuery::Privilege privilege);
} // namespace memgraph::glue::v2

View File

@ -0,0 +1,277 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/v2/communication.hpp"
#include <map>
#include <string>
#include <vector>
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "utils/temporal.hpp"
using memgraph::communication::bolt::Value;
namespace memgraph::glue::v2 {
query::v2::TypedValue ToTypedValue(const Value &value) {
switch (value.type()) {
case Value::Type::Null:
return {};
case Value::Type::Bool:
return query::v2::TypedValue(value.ValueBool());
case Value::Type::Int:
return query::v2::TypedValue(value.ValueInt());
case Value::Type::Double:
return query::v2::TypedValue(value.ValueDouble());
case Value::Type::String:
return query::v2::TypedValue(value.ValueString());
case Value::Type::List: {
std::vector<query::v2::TypedValue> list;
list.reserve(value.ValueList().size());
for (const auto &v : value.ValueList()) list.push_back(ToTypedValue(v));
return query::v2::TypedValue(std::move(list));
}
case Value::Type::Map: {
std::map<std::string, query::v2::TypedValue> map;
for (const auto &kv : value.ValueMap()) map.emplace(kv.first, ToTypedValue(kv.second));
return query::v2::TypedValue(std::move(map));
}
case Value::Type::Vertex:
case Value::Type::Edge:
case Value::Type::UnboundedEdge:
case Value::Type::Path:
throw communication::bolt::ValueException("Unsupported conversion from Value to TypedValue");
case Value::Type::Date:
return query::v2::TypedValue(value.ValueDate());
case Value::Type::LocalTime:
return query::v2::TypedValue(value.ValueLocalTime());
case Value::Type::LocalDateTime:
return query::v2::TypedValue(value.ValueLocalDateTime());
case Value::Type::Duration:
return query::v2::TypedValue(value.ValueDuration());
}
}
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const query::v2::VertexAccessor &vertex,
const storage::v3::Shard &db, storage::v3::View view) {
return ToBoltVertex(vertex.impl_, db, view);
}
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const query::v2::EdgeAccessor &edge,
const storage::v3::Shard &db, storage::v3::View view) {
return ToBoltEdge(edge.impl_, db, view);
}
storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value, const storage::v3::Shard &db,
storage::v3::View view) {
switch (value.type()) {
case query::v2::TypedValue::Type::Null:
return Value();
case query::v2::TypedValue::Type::Bool:
return Value(value.ValueBool());
case query::v2::TypedValue::Type::Int:
return Value(value.ValueInt());
case query::v2::TypedValue::Type::Double:
return Value(value.ValueDouble());
case query::v2::TypedValue::Type::String:
return Value(std::string(value.ValueString()));
case query::v2::TypedValue::Type::List: {
std::vector<Value> values;
values.reserve(value.ValueList().size());
for (const auto &v : value.ValueList()) {
auto maybe_value = ToBoltValue(v, db, view);
if (maybe_value.HasError()) return maybe_value.GetError();
values.emplace_back(std::move(*maybe_value));
}
return Value(std::move(values));
}
case query::v2::TypedValue::Type::Map: {
std::map<std::string, Value> map;
for (const auto &kv : value.ValueMap()) {
auto maybe_value = ToBoltValue(kv.second, db, view);
if (maybe_value.HasError()) return maybe_value.GetError();
map.emplace(kv.first, std::move(*maybe_value));
}
return Value(std::move(map));
}
case query::v2::TypedValue::Type::Vertex: {
auto maybe_vertex = ToBoltVertex(value.ValueVertex(), db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
return Value(std::move(*maybe_vertex));
}
case query::v2::TypedValue::Type::Edge: {
auto maybe_edge = ToBoltEdge(value.ValueEdge(), db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
return Value(std::move(*maybe_edge));
}
case query::v2::TypedValue::Type::Path: {
auto maybe_path = ToBoltPath(value.ValuePath(), db, view);
if (maybe_path.HasError()) return maybe_path.GetError();
return Value(std::move(*maybe_path));
}
case query::v2::TypedValue::Type::Date:
return Value(value.ValueDate());
case query::v2::TypedValue::Type::LocalTime:
return Value(value.ValueLocalTime());
case query::v2::TypedValue::Type::LocalDateTime:
return Value(value.ValueLocalDateTime());
case query::v2::TypedValue::Type::Duration:
return Value(value.ValueDuration());
}
}
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const storage::v3::Shard &db, storage::v3::View view) {
// TODO(jbajic) Fix bolt communication
auto id = communication::bolt::Id::FromUint(0);
auto maybe_labels = vertex.Labels(view);
if (maybe_labels.HasError()) return maybe_labels.GetError();
std::vector<std::string> labels;
labels.reserve(maybe_labels->size());
for (const auto &label : *maybe_labels) {
labels.push_back(db.LabelToName(label));
}
auto maybe_properties = vertex.Properties(view);
if (maybe_properties.HasError()) return maybe_properties.GetError();
std::map<std::string, Value> properties;
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
return communication::bolt::Vertex{id, labels, properties};
}
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const storage::v3::Shard &db, storage::v3::View view) {
// TODO(jbajic) Fix bolt communication
auto id = communication::bolt::Id::FromUint(0);
auto from = communication::bolt::Id::FromUint(0);
auto to = communication::bolt::Id::FromUint(0);
const auto &type = db.EdgeTypeToName(edge.EdgeType());
auto maybe_properties = edge.Properties(view);
if (maybe_properties.HasError()) return maybe_properties.GetError();
std::map<std::string, Value> properties;
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
return communication::bolt::Edge{id, from, to, type, properties};
}
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db,
storage::v3::View view) {
std::vector<communication::bolt::Vertex> vertices;
vertices.reserve(path.vertices().size());
for (const auto &v : path.vertices()) {
auto maybe_vertex = ToBoltVertex(v, db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
vertices.emplace_back(std::move(*maybe_vertex));
}
std::vector<communication::bolt::Edge> edges;
edges.reserve(path.edges().size());
for (const auto &e : path.edges()) {
auto maybe_edge = ToBoltEdge(e, db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
edges.emplace_back(std::move(*maybe_edge));
}
return communication::bolt::Path(vertices, edges);
}
storage::v3::PropertyValue ToPropertyValue(const Value &value) {
switch (value.type()) {
case Value::Type::Null:
return {};
case Value::Type::Bool:
return storage::v3::PropertyValue(value.ValueBool());
case Value::Type::Int:
return storage::v3::PropertyValue(value.ValueInt());
case Value::Type::Double:
return storage::v3::PropertyValue(value.ValueDouble());
case Value::Type::String:
return storage::v3::PropertyValue(value.ValueString());
case Value::Type::List: {
std::vector<storage::v3::PropertyValue> vec;
vec.reserve(value.ValueList().size());
for (const auto &value : value.ValueList()) vec.emplace_back(ToPropertyValue(value));
return storage::v3::PropertyValue(std::move(vec));
}
case Value::Type::Map: {
std::map<std::string, storage::v3::PropertyValue> map;
for (const auto &kv : value.ValueMap()) map.emplace(kv.first, ToPropertyValue(kv.second));
return storage::v3::PropertyValue(std::move(map));
}
case Value::Type::Vertex:
case Value::Type::Edge:
case Value::Type::UnboundedEdge:
case Value::Type::Path:
throw communication::bolt::ValueException("Unsupported conversion from Value to PropertyValue");
case Value::Type::Date:
return storage::v3::PropertyValue(
storage::v3::TemporalData(storage::v3::TemporalType::Date, value.ValueDate().MicrosecondsSinceEpoch()));
case Value::Type::LocalTime:
return storage::v3::PropertyValue(storage::v3::TemporalData(storage::v3::TemporalType::LocalTime,
value.ValueLocalTime().MicrosecondsSinceEpoch()));
case Value::Type::LocalDateTime:
return storage::v3::PropertyValue(storage::v3::TemporalData(storage::v3::TemporalType::LocalDateTime,
value.ValueLocalDateTime().MicrosecondsSinceEpoch()));
case Value::Type::Duration:
return storage::v3::PropertyValue(
storage::v3::TemporalData(storage::v3::TemporalType::Duration, value.ValueDuration().microseconds));
}
}
Value ToBoltValue(const storage::v3::PropertyValue &value) {
switch (value.type()) {
case storage::v3::PropertyValue::Type::Null:
return {};
case storage::v3::PropertyValue::Type::Bool:
return {value.ValueBool()};
case storage::v3::PropertyValue::Type::Int:
return {value.ValueInt()};
break;
case storage::v3::PropertyValue::Type::Double:
return {value.ValueDouble()};
case storage::v3::PropertyValue::Type::String:
return {value.ValueString()};
case storage::v3::PropertyValue::Type::List: {
const auto &values = value.ValueList();
std::vector<Value> vec;
vec.reserve(values.size());
for (const auto &v : values) {
vec.push_back(ToBoltValue(v));
}
return {std::move(vec)};
}
case storage::v3::PropertyValue::Type::Map: {
const auto &map = value.ValueMap();
std::map<std::string, Value> dv_map;
for (const auto &kv : map) {
dv_map.emplace(kv.first, ToBoltValue(kv.second));
}
return {std::move(dv_map)};
}
case storage::v3::PropertyValue::Type::TemporalData:
const auto &type = value.ValueTemporalData();
switch (type.type) {
case storage::v3::TemporalType::Date:
return {utils::Date(type.microseconds)};
case storage::v3::TemporalType::LocalTime:
return {utils::LocalTime(type.microseconds)};
case storage::v3::TemporalType::LocalDateTime:
return {utils::LocalDateTime(type.microseconds)};
case storage::v3::TemporalType::Duration:
return {utils::Duration(type.microseconds)};
}
}
}
} // namespace memgraph::glue::v2

View File

@ -0,0 +1,68 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file Conversion functions between Value and other memgraph types.
#pragma once
#include "communication/bolt/v1/value.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/view.hpp"
namespace memgraph::storage::v3 {
class EdgeAccessor;
class Storage;
class VertexAccessor;
} // namespace memgraph::storage::v3
namespace memgraph::glue::v2 {
/// @param storage::v3::VertexAccessor for converting to
/// communication::bolt::Vertex.
/// @param storage::v3::Shard for getting label and property names.
/// @param storage::v3::View for deciding which vertex attributes are visible.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const storage::v3::Shard &db, storage::v3::View view);
/// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge.
/// @param storage::v3::Shard for getting edge type and property names.
/// @param storage::v3::View for deciding which edge attributes are visible.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const storage::v3::Shard &db, storage::v3::View view);
/// @param query::v2::Path for converting to communication::bolt::Path.
/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db,
storage::v3::View view);
/// @param query::v2::TypedValue for converting to communication::bolt::Value.
/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Value> ToBoltValue(const query::v2::TypedValue &value,
const storage::v3::Shard &db, storage::v3::View view);
query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value);
communication::bolt::Value ToBoltValue(const storage::v3::PropertyValue &value);
storage::v3::PropertyValue ToPropertyValue(const communication::bolt::Value &value);
} // namespace memgraph::glue::v2

77
src/io/address.hpp Normal file
View File

@ -0,0 +1,77 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <compare>
#include <fmt/format.h>
#include <boost/asio/ip/tcp.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>
namespace memgraph::io {
struct Address {
// It's important for all participants to have a
// unique identifier - IP and port alone are not
// enough, and may change over the lifecycle of
// the nodes. Particularly storage nodes may change
// their IP addresses over time, and the system
// should gracefully update its information
// about them.
boost::uuids::uuid unique_id;
boost::asio::ip::address last_known_ip;
uint16_t last_known_port;
static Address TestAddress(uint16_t port) {
return Address{
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
.last_known_port = port,
};
}
static Address UniqueLocalAddress() {
return Address{
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
};
}
/// Returns a new ID with the same IP and port but a unique UUID.
Address ForkUniqueAddress() {
return Address{
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
.last_known_ip = last_known_ip,
.last_known_port = last_known_port,
};
}
friend bool operator==(const Address &lhs, const Address &rhs) = default;
/// unique_id is most dominant for ordering, then last_known_ip, then last_known_port
friend bool operator<(const Address &lhs, const Address &rhs) {
if (lhs.unique_id != rhs.unique_id) {
return lhs.unique_id < rhs.unique_id;
}
if (lhs.last_known_ip != rhs.last_known_ip) {
return lhs.last_known_ip < rhs.last_known_ip;
}
return lhs.last_known_port < rhs.last_known_port;
}
std::string ToString() const {
return fmt::format("Address {{ unique_id: {}, last_known_ip: {}, last_known_port: {} }}",
boost::uuids::to_string(unique_id), last_known_ip.to_string(), last_known_port);
}
};
}; // namespace memgraph::io

26
src/io/errors.hpp Normal file
View File

@ -0,0 +1,26 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
namespace memgraph::io {
// Signifies that a retriable operation was unable to
// complete after a configured number of retries.
struct RetriesExhausted {};
// Signifies that a request was unable to receive a response
// within some configured timeout duration. It is important
// to remember that in distributed systems, a timeout does
// not signify that a request was not received or processed.
// It may be the case that the request was fully processed
// but that the response was not received.
struct TimedOut {};
}; // namespace memgraph::io

256
src/io/future.hpp Normal file
View File

@ -0,0 +1,256 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <condition_variable>
#include <memory>
#include <mutex>
#include <optional>
#include <thread>
#include <utility>
#include "io/errors.hpp"
#include "utils/logging.hpp"
namespace memgraph::io {
// Shared is in an anonymous namespace, and the only way to
// construct a Promise or Future is to pass a Shared in. This
// ensures that Promises and Futures can only be constructed
// in this translation unit.
namespace details {
template <typename T>
class Shared {
mutable std::condition_variable cv_;
mutable std::mutex mu_;
std::optional<T> item_;
bool consumed_ = false;
bool waiting_ = false;
std::function<bool()> simulator_notifier_ = nullptr;
public:
explicit Shared(std::function<bool()> simulator_notifier) : simulator_notifier_(simulator_notifier) {}
Shared() = default;
Shared(Shared &&) = delete;
Shared &operator=(Shared &&) = delete;
Shared(const Shared &) = delete;
Shared &operator=(const Shared &) = delete;
~Shared() = default;
/// Takes the item out of our optional item_ and returns it.
T Take() {
MG_ASSERT(item_, "Take called without item_ being present");
MG_ASSERT(!consumed_, "Take called on already-consumed Future");
T ret = std::move(item_).value();
item_.reset();
consumed_ = true;
return ret;
}
T Wait() {
std::unique_lock<std::mutex> lock(mu_);
waiting_ = true;
while (!item_) {
bool simulator_progressed = false;
if (simulator_notifier_) [[unlikely]] {
// We can't hold our own lock while notifying
// the simulator because notifying the simulator
// involves acquiring the simulator's mutex
// to guarantee that our notification linearizes
// with the simulator's condition variable.
// However, the simulator may acquire our
// mutex to check if we are being awaited,
// while determining system quiescence,
// so we have to get out of its way to avoid
// a cyclical deadlock.
lock.unlock();
simulator_progressed = std::invoke(simulator_notifier_);
lock.lock();
if (item_) {
// item may have been filled while we
// had dropped our mutex while notifying
// the simulator of our waiting_ status.
break;
}
}
if (!simulator_progressed) [[likely]] {
cv_.wait(lock);
}
MG_ASSERT(!consumed_, "Future consumed twice!");
}
waiting_ = false;
return Take();
}
bool IsReady() const {
std::unique_lock<std::mutex> lock(mu_);
return item_.has_value();
}
std::optional<T> TryGet() {
std::unique_lock<std::mutex> lock(mu_);
if (item_) {
return Take();
}
return std::nullopt;
}
void Fill(T item) {
{
std::unique_lock<std::mutex> lock(mu_);
MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
MG_ASSERT(!item_, "Promise filled twice!");
item_ = item;
} // lock released before condition variable notification
cv_.notify_all();
}
bool IsAwaited() const {
std::unique_lock<std::mutex> lock(mu_);
return waiting_;
}
};
} // namespace details
template <typename T>
class Future {
bool consumed_or_moved_ = false;
std::shared_ptr<details::Shared<T>> shared_;
public:
explicit Future(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
Future() = delete;
Future(Future &&old) noexcept {
MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed.");
shared_ = std::move(old.shared_);
consumed_or_moved_ = old.consumed_or_moved_;
old.consumed_or_moved_ = true;
}
Future(const Future &) = delete;
Future &operator=(const Future &) = delete;
~Future() = default;
/// Returns true if the Future is ready to
/// be consumed using TryGet or Wait (prefer Wait
/// if you know it's ready, because it doesn't
/// return an optional.
bool IsReady() {
MG_ASSERT(!consumed_or_moved_, "Called IsReady after Future already consumed!");
return shared_->IsReady();
}
/// Non-blocking method that returns the inner
/// item if it's already ready, or std::nullopt
/// if it is not ready yet.
std::optional<T> TryGet() {
MG_ASSERT(!consumed_or_moved_, "Called TryGet after Future already consumed!");
std::optional<T> ret = shared_->TryGet();
if (ret) {
consumed_or_moved_ = true;
}
return ret;
}
/// Block on the corresponding promise to be filled,
/// returning the inner item when ready.
T Wait() && {
MG_ASSERT(!consumed_or_moved_, "Future should only be consumed with Wait once!");
T ret = shared_->Wait();
consumed_or_moved_ = true;
return ret;
}
/// Marks this Future as canceled.
void Cancel() {
MG_ASSERT(!consumed_or_moved_, "Future::Cancel called on a future that was already moved or consumed!");
consumed_or_moved_ = true;
}
};
template <typename T>
class Promise {
std::shared_ptr<details::Shared<T>> shared_;
bool filled_or_moved_ = false;
public:
explicit Promise(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
Promise() = delete;
Promise(Promise &&old) noexcept {
MG_ASSERT(!old.filled_or_moved_, "Promise moved from after already being moved from or filled.");
shared_ = std::move(old.shared_);
old.filled_or_moved_ = true;
}
Promise &operator=(Promise &&old) noexcept {
MG_ASSERT(!old.filled_or_moved_, "Promise moved from after already being moved from or filled.");
shared_ = std::move(old.shared_);
old.filled_or_moved_ = true;
}
Promise(const Promise &) = delete;
Promise &operator=(const Promise &) = delete;
~Promise() { MG_ASSERT(filled_or_moved_, "Promise destroyed before its associated Future was filled!"); }
// Fill the expected item into the Future.
void Fill(T item) {
MG_ASSERT(!filled_or_moved_, "Promise::Fill called on a promise that is already filled or moved!");
shared_->Fill(item);
filled_or_moved_ = true;
}
bool IsAwaited() { return shared_->IsAwaited(); }
/// Moves this Promise into a unique_ptr.
std::unique_ptr<Promise<T>> ToUnique() && {
std::unique_ptr<Promise<T>> up = std::make_unique<Promise<T>>(std::move(shared_));
filled_or_moved_ = true;
return up;
}
};
template <typename T>
std::pair<Future<T>, Promise<T>> FuturePromisePair() {
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>();
Future<T> future = Future<T>(shared);
Promise<T> promise = Promise<T>(shared);
return std::make_pair(std::move(future), std::move(promise));
}
template <typename T>
std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifier(std::function<bool()> simulator_notifier) {
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(simulator_notifier);
Future<T> future = Future<T>(shared);
Promise<T> promise = Promise<T>(shared);
return std::make_pair(std::move(future), std::move(promise));
}
}; // namespace memgraph::io

View File

@ -0,0 +1,35 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <random>
#include "io/address.hpp"
#include "io/local_transport/local_transport.hpp"
#include "io/local_transport/local_transport_handle.hpp"
#include "io/transport.hpp"
namespace memgraph::io::local_transport {
class LocalSystem {
std::shared_ptr<LocalTransportHandle> local_transport_handle_ = std::make_shared<LocalTransportHandle>();
public:
Io<LocalTransport> Register(Address address) {
LocalTransport local_transport(local_transport_handle_);
return Io{local_transport, address};
}
void ShutDown() { local_transport_handle_->ShutDown(); }
};
} // namespace memgraph::io::local_transport

View File

@ -0,0 +1,64 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <memory>
#include <random>
#include <utility>
#include "io/address.hpp"
#include "io/local_transport/local_transport_handle.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
namespace memgraph::io::local_transport {
class LocalTransport {
std::shared_ptr<LocalTransportHandle> local_transport_handle_;
public:
explicit LocalTransport(std::shared_ptr<LocalTransportHandle> local_transport_handle)
: local_transport_handle_(std::move(local_transport_handle)) {}
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestId request_id, RequestT request,
Duration timeout) {
auto [future, promise] = memgraph::io::FuturePromisePair<ResponseResult<ResponseT>>();
local_transport_handle_->SubmitRequest(to_address, from_address, request_id, std::move(request), timeout,
std::move(promise));
return std::move(future);
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address receiver_address, Duration timeout) {
return local_transport_handle_->template Receive<Ms...>(receiver_address, timeout);
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
}
Time Now() const { return local_transport_handle_->Now(); }
bool ShouldShutDown() const { return local_transport_handle_->ShouldShutDown(); }
template <class D = std::poisson_distribution<>, class Return = uint64_t>
Return Rand(D distrib) {
std::random_device rng;
return distrib(rng);
}
};
}; // namespace memgraph::io::local_transport

View File

@ -0,0 +1,151 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <condition_variable>
#include <iostream>
#include <map>
#include <mutex>
#include "io/errors.hpp"
#include "io/message_conversion.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
namespace memgraph::io::local_transport {
class LocalTransportHandle {
mutable std::mutex mu_{};
mutable std::condition_variable cv_;
bool should_shut_down_ = false;
// the responses to requests that are being waited on
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
// messages that are sent to servers that may later receive them
std::vector<OpaqueMessage> can_receive_;
public:
~LocalTransportHandle() {
for (auto &&[pk, promise] : promises_) {
std::move(promise.promise).TimeOut();
}
promises_.clear();
}
void ShutDown() {
std::unique_lock<std::mutex> lock(mu_);
should_shut_down_ = true;
cv_.notify_all();
}
bool ShouldShutDown() const {
std::unique_lock<std::mutex> lock(mu_);
return should_shut_down_;
}
static Time Now() {
auto nano_time = std::chrono::system_clock::now();
return std::chrono::time_point_cast<std::chrono::microseconds>(nano_time);
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address /* receiver_address */, Duration timeout) {
std::unique_lock lock(mu_);
Time before = Now();
spdlog::info("can_receive_ size: {}", can_receive_.size());
while (can_receive_.empty()) {
Time now = Now();
// protection against non-monotonic timesources
auto maxed_now = std::max(now, before);
auto elapsed = maxed_now - before;
if (timeout < elapsed) {
return TimedOut{};
}
Duration relative_timeout = timeout - elapsed;
std::cv_status cv_status_value = cv_.wait_for(lock, relative_timeout);
if (cv_status_value == std::cv_status::timeout) {
return TimedOut{};
}
}
auto current_message = std::move(can_receive_.back());
can_receive_.pop_back();
auto m_opt = std::move(current_message).Take<Ms...>();
return std::move(m_opt).value();
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
std::any message_any(std::forward<M>(message));
OpaqueMessage opaque_message{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message_any)};
PromiseKey promise_key{
.requester_address = to_address, .request_id = opaque_message.request_id, .replier_address = from_address};
{
std::unique_lock<std::mutex> lock(mu_);
if (promises_.contains(promise_key)) {
spdlog::info("using message to fill promise");
// complete waiting promise if it's there
DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
promises_.erase(promise_key);
dop.promise.Fill(std::move(opaque_message));
} else {
spdlog::info("placing message in can_receive_");
can_receive_.emplace_back(std::move(opaque_message));
}
} // lock dropped
cv_.notify_all();
}
template <Message RequestT, Message ResponseT>
void SubmitRequest(Address to_address, Address from_address, RequestId request_id, RequestT &&request,
Duration timeout, ResponsePromise<ResponseT> promise) {
const bool port_matches = to_address.last_known_port == from_address.last_known_port;
const bool ip_matches = to_address.last_known_ip == from_address.last_known_ip;
MG_ASSERT(port_matches && ip_matches);
const Time deadline = Now() + timeout;
{
std::unique_lock<std::mutex> lock(mu_);
PromiseKey promise_key{
.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
OpaquePromise opaque_promise(std::move(promise).ToUnique());
DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
promises_.emplace(std::move(promise_key), std::move(dop));
} // lock dropped
Send(to_address, from_address, request_id, std::forward<RequestT>(request));
}
};
} // namespace memgraph::io::local_transport

View File

@ -0,0 +1,232 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "io/transport.hpp"
namespace memgraph::io {
using memgraph::io::Duration;
using memgraph::io::Message;
using memgraph::io::Time;
struct PromiseKey {
Address requester_address;
uint64_t request_id;
// TODO(tyler) possibly remove replier_address from promise key
// once we want to support DSR.
Address replier_address;
public:
friend bool operator<(const PromiseKey &lhs, const PromiseKey &rhs) {
if (lhs.requester_address != rhs.requester_address) {
return lhs.requester_address < rhs.requester_address;
}
if (lhs.request_id != rhs.request_id) {
return lhs.request_id < rhs.request_id;
}
return lhs.replier_address < rhs.replier_address;
}
};
struct OpaqueMessage {
Address to_address;
Address from_address;
uint64_t request_id;
std::any message;
/// Recursively tries to match a specific type from the outer
/// variant's parameter pack against the type of the std::any,
/// and if it matches, make it concrete and return it. Otherwise,
/// move on and compare the any with the next type from the
/// parameter pack.
///
/// Return is the full std::variant<Ts...> type that holds the
/// full parameter pack without interfering with recursive
/// narrowing expansion.
template <typename Return, Message Head, Message... Rest>
std::optional<Return> Unpack(std::any &&a) {
if (typeid(Head) == a.type()) {
Head concrete = std::any_cast<Head>(std::move(a));
return concrete;
}
if constexpr (sizeof...(Rest) > 0) {
return Unpack<Return, Rest...>(std::move(a));
} else {
return std::nullopt;
}
}
/// High level "user-facing" conversion function that lets
/// people interested in conversion only supply a single
/// parameter pack for the types that they want to compare
/// with the any and potentially include in the returned
/// variant.
template <Message... Ms>
requires(sizeof...(Ms) > 0) std::optional<std::variant<Ms...>> VariantFromAny(std::any &&a) {
return Unpack<std::variant<Ms...>, Ms...>(std::move(a));
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) std::optional<RequestEnvelope<Ms...>> Take() && {
std::optional<std::variant<Ms...>> m_opt = VariantFromAny<Ms...>(std::move(message));
if (m_opt) {
return RequestEnvelope<Ms...>{
.message = std::move(*m_opt),
.request_id = request_id,
.to_address = to_address,
.from_address = from_address,
};
}
return std::nullopt;
}
};
class OpaquePromiseTraitBase {
public:
virtual const std::type_info *TypeInfo() const = 0;
virtual bool IsAwaited(void *ptr) const = 0;
virtual void Fill(void *ptr, OpaqueMessage &&) const = 0;
virtual void TimeOut(void *ptr) const = 0;
virtual ~OpaquePromiseTraitBase() = default;
OpaquePromiseTraitBase() = default;
OpaquePromiseTraitBase(const OpaquePromiseTraitBase &) = delete;
OpaquePromiseTraitBase &operator=(const OpaquePromiseTraitBase &) = delete;
OpaquePromiseTraitBase(OpaquePromiseTraitBase &&old) = delete;
OpaquePromiseTraitBase &operator=(OpaquePromiseTraitBase &&) = delete;
};
template <typename T>
class OpaquePromiseTrait : public OpaquePromiseTraitBase {
public:
const std::type_info *TypeInfo() const override { return &typeid(T); };
bool IsAwaited(void *ptr) const override { return static_cast<ResponsePromise<T> *>(ptr)->IsAwaited(); };
void Fill(void *ptr, OpaqueMessage &&opaque_message) const override {
T message = std::any_cast<T>(std::move(opaque_message.message));
auto response_envelope = ResponseEnvelope<T>{.message = std::move(message),
.request_id = opaque_message.request_id,
.to_address = opaque_message.to_address,
.from_address = opaque_message.from_address};
auto promise = static_cast<ResponsePromise<T> *>(ptr);
auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
unique_promise->Fill(std::move(response_envelope));
};
void TimeOut(void *ptr) const override {
auto promise = static_cast<ResponsePromise<T> *>(ptr);
auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
ResponseResult<T> result = TimedOut{};
unique_promise->Fill(std::move(result));
}
};
class OpaquePromise {
void *ptr_;
std::unique_ptr<OpaquePromiseTraitBase> trait_;
public:
OpaquePromise(OpaquePromise &&old) noexcept : ptr_(old.ptr_), trait_(std::move(old.trait_)) {
MG_ASSERT(old.ptr_ != nullptr);
old.ptr_ = nullptr;
}
OpaquePromise &operator=(OpaquePromise &&old) noexcept {
MG_ASSERT(ptr_ == nullptr);
MG_ASSERT(old.ptr_ != nullptr);
MG_ASSERT(this != &old);
ptr_ = old.ptr_;
trait_ = std::move(old.trait_);
old.ptr_ = nullptr;
return *this;
}
OpaquePromise(const OpaquePromise &) = delete;
OpaquePromise &operator=(const OpaquePromise &) = delete;
template <typename T>
std::unique_ptr<ResponsePromise<T>> Take() && {
MG_ASSERT(typeid(T) == *trait_->TypeInfo());
MG_ASSERT(ptr_ != nullptr);
auto ptr = static_cast<ResponsePromise<T> *>(ptr_);
ptr_ = nullptr;
return std::unique_ptr<T>(ptr);
}
template <typename T>
explicit OpaquePromise(std::unique_ptr<ResponsePromise<T>> promise)
: ptr_(static_cast<void *>(promise.release())), trait_(std::make_unique<OpaquePromiseTrait<T>>()) {}
bool IsAwaited() {
MG_ASSERT(ptr_ != nullptr);
return trait_->IsAwaited(ptr_);
}
void TimeOut() {
MG_ASSERT(ptr_ != nullptr);
trait_->TimeOut(ptr_);
ptr_ = nullptr;
}
void Fill(OpaqueMessage &&opaque_message) {
MG_ASSERT(ptr_ != nullptr);
trait_->Fill(ptr_, std::move(opaque_message));
ptr_ = nullptr;
}
~OpaquePromise() {
MG_ASSERT(ptr_ == nullptr, "OpaquePromise destroyed without being explicitly timed out or filled");
}
};
struct DeadlineAndOpaquePromise {
Time deadline;
OpaquePromise promise;
};
template <class From>
std::type_info const &type_info_for_variant(From const &from) {
return std::visit([](auto &&x) -> decltype(auto) { return typeid(x); }, from);
}
template <typename From, typename Return, typename Head, typename... Rest>
std::optional<Return> ConvertVariantInner(From &&a) {
if (typeid(Head) == type_info_for_variant(a)) {
Head concrete = std::get<Head>(std::forward<From>(a));
return concrete;
}
if constexpr (sizeof...(Rest) > 0) {
return ConvertVariantInner<From, Return, Rest...>(std::forward<From>(a));
} else {
return std::nullopt;
}
}
/// This function converts a variant to another variant holding a subset OR superset of
/// possible types.
template <class From, class... Ms>
requires(sizeof...(Ms) > 0) std::optional<std::variant<Ms...>> ConvertVariant(From &&from) {
return ConvertVariantInner<From, std::variant<Ms...>, Ms...>(std::forward<From>(from));
}
} // namespace memgraph::io

46
src/io/messages.hpp Normal file
View File

@ -0,0 +1,46 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <variant>
#include <coordinator/coordinator.hpp>
#include <io/rsm/raft.hpp>
#include "query/v2/requests.hpp"
#include "utils/concepts.hpp"
namespace memgraph::io::messages {
using memgraph::coordinator::CoordinatorReadRequests;
using memgraph::coordinator::CoordinatorWriteRequests;
using memgraph::coordinator::CoordinatorWriteResponses;
using StorageReadRequest = msgs::ReadRequests;
using StorageWriteRequest = msgs::WriteRequests;
using memgraph::io::rsm::AppendRequest;
using memgraph::io::rsm::AppendResponse;
using memgraph::io::rsm::ReadRequest;
using memgraph::io::rsm::VoteRequest;
using memgraph::io::rsm::VoteResponse;
using memgraph::io::rsm::WriteRequest;
using memgraph::io::rsm::WriteResponse;
using CoordinatorMessages =
std::variant<ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>, AppendResponse,
WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse>;
using ShardMessages = std::variant<ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>, AppendResponse,
WriteRequest<StorageWriteRequest>, VoteRequest, VoteResponse>;
using ShardManagerMessages = std::variant<WriteResponse<CoordinatorWriteResponses>>;
} // namespace memgraph::io::messages

926
src/io/rsm/raft.hpp Normal file
View File

@ -0,0 +1,926 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
// TODO(tyler) buffer out-of-order Append buffers on the Followers to reassemble more quickly
// TODO(tyler) handle granular batch sizes based on simple flow control
#pragma once
#include <deque>
#include <iostream>
#include <map>
#include <set>
#include <thread>
#include <unordered_map>
#include <vector>
#include "io/simulator/simulator.hpp"
#include "io/transport.hpp"
#include "utils/concepts.hpp"
namespace memgraph::io::rsm {
/// Timeout and replication tunables
using namespace std::chrono_literals;
static constexpr auto kMinimumElectionTimeout = 100ms;
static constexpr auto kMaximumElectionTimeout = 200ms;
static constexpr auto kMinimumBroadcastTimeout = 40ms;
static constexpr auto kMaximumBroadcastTimeout = 60ms;
static constexpr auto kMinimumCronInterval = 1ms;
static constexpr auto kMaximumCronInterval = 2ms;
static constexpr auto kMinimumReceiveTimeout = 40ms;
static constexpr auto kMaximumReceiveTimeout = 60ms;
static_assert(kMinimumElectionTimeout > kMaximumBroadcastTimeout,
"The broadcast timeout has to be smaller than the election timeout!");
static_assert(kMinimumElectionTimeout < kMaximumElectionTimeout,
"The minimum election timeout has to be smaller than the maximum election timeout!");
static_assert(kMinimumBroadcastTimeout < kMaximumBroadcastTimeout,
"The minimum broadcast timeout has to be smaller than the maximum broadcast timeout!");
static_assert(kMinimumCronInterval < kMaximumCronInterval,
"The minimum cron interval has to be smaller than the maximum cron interval!");
static_assert(kMinimumReceiveTimeout < kMaximumReceiveTimeout,
"The minimum receive timeout has to be smaller than the maximum receive timeout!");
static constexpr size_t kMaximumAppendBatchSize = 1024;
using Term = uint64_t;
using LogIndex = uint64_t;
using LogSize = uint64_t;
template <typename WriteOperation>
struct WriteRequest {
WriteOperation operation;
};
/// WriteResponse is returned to a client after
/// their WriteRequest was entered in to the raft
/// log and it reached consensus.
///
/// WriteReturn is the result of applying the WriteRequest to
/// ReplicatedState, and if the ReplicatedState::write
/// method is deterministic, all replicas will
/// have the same ReplicatedState after applying
/// the submitted WriteRequest.
template <typename WriteReturn>
struct WriteResponse {
bool success;
WriteReturn write_return;
std::optional<Address> retry_leader;
LogIndex raft_index;
};
template <typename ReadOperation>
struct ReadRequest {
ReadOperation operation;
};
template <typename ReadReturn>
struct ReadResponse {
bool success;
ReadReturn read_return;
std::optional<Address> retry_leader;
};
/// AppendRequest is a raft-level message that the Leader
/// periodically broadcasts to all Follower peers. This
/// serves three main roles:
/// 1. acts as a heartbeat from the Leader to the Follower
/// 2. replicates new data that the Leader has received to the Follower
/// 3. informs Follower peers when the commit index has increased,
/// signalling that it is now safe to apply log items to the
/// replicated state machine
template <typename WriteRequest>
struct AppendRequest {
Term term = 0;
LogIndex batch_start_log_index;
Term last_log_term;
std::vector<std::pair<Term, WriteRequest>> entries;
LogSize leader_commit;
};
struct AppendResponse {
bool success;
Term term;
Term last_log_term;
// a small optimization over the raft paper, tells
// the leader the offset that we are interested in
// to send log offsets from for us. This will only
// be useful at the beginning of a leader's term.
LogSize log_size;
};
struct VoteRequest {
Term term = 0;
LogSize log_size;
Term last_log_term;
};
struct VoteResponse {
Term term = 0;
LogSize committed_log_size;
bool vote_granted = false;
};
template <typename WriteRequest>
struct CommonState {
Term term = 0;
std::vector<std::pair<Term, WriteRequest>> log;
LogSize committed_log_size = 0;
LogSize applied_size = 0;
};
struct FollowerTracker {
LogIndex next_index = 0;
LogSize confirmed_log_size = 0;
};
struct PendingClientRequest {
RequestId request_id;
Address address;
Time received_at;
};
struct Leader {
std::map<Address, FollowerTracker> followers;
std::unordered_map<LogIndex, PendingClientRequest> pending_client_requests;
Time last_broadcast = Time::min();
std::string static ToString() { return "\tLeader \t"; }
};
struct Candidate {
std::map<Address, LogSize> successful_votes;
Time election_began = Time::min();
std::set<Address> outstanding_votes;
std::string static ToString() { return "\tCandidate\t"; }
};
struct Follower {
Time last_received_append_entries_timestamp;
Address leader_address;
std::string static ToString() { return "\tFollower \t"; }
};
using Role = std::variant<Candidate, Leader, Follower>;
template <typename Role>
concept AllRoles = memgraph::utils::SameAsAnyOf<Role, Leader, Follower, Candidate>;
template <typename Role>
concept LeaderOrFollower = memgraph::utils::SameAsAnyOf<Role, Leader, Follower>;
template <typename Role>
concept FollowerOrCandidate = memgraph::utils::SameAsAnyOf<Role, Follower, Candidate>;
/*
all ReplicatedState classes should have an Apply method
that returns our WriteResponseValue after consensus, and
a Read method that returns our ReadResponseValue without
requiring consensus.
ReadResponse Read(ReadOperation);
WriteResponseValue ReplicatedState::Apply(WriteRequest);
For example:
If the state is uint64_t, and WriteRequest is `struct PlusOne {};`,
and WriteResponseValue is also uint64_t (the new value), then
each call to state.Apply(PlusOne{}) will return the new value
after incrementing it. 0, 1, 2, 3... and this will be sent back
to the client that requested the mutation.
In practice, these mutations will usually be predicated on some
previous value, so that they are idempotent, functioning similarly
to a CAS operation.
*/
template <typename WriteOperation, typename ReadOperation, typename ReplicatedState, typename WriteResponseValue,
typename ReadResponseValue>
concept Rsm = requires(ReplicatedState state, WriteOperation w, ReadOperation r) {
{ state.Read(r) } -> std::same_as<ReadResponseValue>;
{ state.Apply(w) } -> std::same_as<WriteResponseValue>;
};
/// Parameter Purpose
/// --------------------------
/// IoImpl the concrete Io provider - SimulatorTransport, ThriftTransport, etc...
/// ReplicatedState the high-level data structure that is managed by the raft-backed replicated state machine
/// WriteOperation the individual operation type that is applied to the ReplicatedState in identical order
/// across each replica
/// WriteResponseValue the return value of calling ReplicatedState::Apply(WriteOperation), which is executed in
/// identical order across all replicas after an WriteOperation reaches consensus.
/// ReadOperation the type of operations that do not require consensus before executing directly
/// on a const ReplicatedState &
/// ReadResponseValue the return value of calling ReplicatedState::Read(ReadOperation), which is executed directly
/// without going through consensus first
template <typename IoImpl, typename ReplicatedState, typename WriteOperation, typename WriteResponseValue,
typename ReadOperation, typename ReadResponseValue>
requires Rsm<WriteOperation, ReadOperation, ReplicatedState, WriteResponseValue, ReadResponseValue>
class Raft {
CommonState<WriteOperation> state_;
Role role_ = Candidate{};
Io<IoImpl> io_;
std::vector<Address> peers_;
ReplicatedState replicated_state_;
Time next_cron_;
public:
Raft(Io<IoImpl> &&io, std::vector<Address> peers, ReplicatedState &&replicated_state)
: io_(std::forward<Io<IoImpl>>(io)),
peers_(peers),
replicated_state_(std::forward<ReplicatedState>(replicated_state)) {
if (peers.empty()) {
role_ = Leader{};
}
}
/// Periodic protocol maintenance. Returns the time that Cron should be called again
/// in the future.
Time Cron() {
// dispatch periodic logic based on our role to a specific Cron method.
std::optional<Role> new_role = std::visit([&](auto &role) { return Cron(role); }, role_);
if (new_role) {
role_ = std::move(new_role).value();
}
const Duration random_cron_interval = RandomTimeout(kMinimumCronInterval, kMaximumCronInterval);
return io_.Now() + random_cron_interval;
}
/// Returns the Address for our underlying Io implementation
Address GetAddress() { return io_.GetAddress(); }
using ReceiveVariant = std::variant<ReadRequest<ReadOperation>, AppendRequest<WriteOperation>, AppendResponse,
WriteRequest<WriteOperation>, VoteRequest, VoteResponse>;
void Handle(ReceiveVariant &&message_variant, RequestId request_id, Address from_address) {
// dispatch the message to a handler based on our role,
// which can be specified in the Handle first argument,
// or it can be `auto` if it's a handler for several roles
// or messages.
std::optional<Role> new_role = std::visit(
[&](auto &&msg, auto &role) mutable {
return Handle(role, std::forward<decltype(msg)>(msg), request_id, from_address);
},
std::forward<ReceiveVariant>(message_variant), role_);
// TODO(tyler) (M3) maybe replace std::visit with get_if for explicit prioritized matching, [[likely]] etc...
if (new_role) {
role_ = std::move(new_role).value();
}
}
void Run() {
while (!io_.ShouldShutDown()) {
const auto now = io_.Now();
if (now >= next_cron_) {
next_cron_ = Cron();
}
const Duration receive_timeout = RandomTimeout(kMinimumReceiveTimeout, kMaximumReceiveTimeout);
auto request_result =
io_.template ReceiveWithTimeout<ReadRequest<ReadOperation>, AppendRequest<WriteOperation>, AppendResponse,
WriteRequest<WriteOperation>, VoteRequest, VoteResponse>(receive_timeout);
if (request_result.HasError()) {
continue;
}
auto request = std::move(request_result.GetValue());
Handle(std::move(request.message), request.request_id, request.from_address);
}
}
private:
// Raft paper - 5.3
// When the entry has been safely replicated, the leader applies the
// entry to its state machine and returns the result of that
// execution to the client.
//
// "Safely replicated" is defined as being known to be present
// on at least a majority of all peers (inclusive of the Leader).
void BumpCommitIndexAndReplyToClients(Leader &leader) {
auto confirmed_log_sizes = std::vector<LogSize>{};
// We include our own log size in the calculation of the log
// confirmed log size that is present on at least a majority of all peers.
confirmed_log_sizes.push_back(state_.log.size());
for (const auto &[addr, f] : leader.followers) {
confirmed_log_sizes.push_back(f.confirmed_log_size);
Log("Follower at port ", addr.last_known_port, " has confirmed log size of: ", f.confirmed_log_size);
}
// reverse sort from highest to lowest (using std::ranges::greater)
std::ranges::sort(confirmed_log_sizes, std::ranges::greater());
// This is a particularly correctness-critical calculation because it
// determines the committed log size that will be broadcast in
// the next AppendRequest.
//
// If the following sizes are recorded for clusters of different numbers of peers,
// these are the expected sizes that are considered to have reached consensus:
//
// state | expected value | (confirmed_log_sizes.size() / 2)
// [1] 1 (1 / 2) => 0
// [2, 1] 1 (2 / 2) => 1
// [3, 2, 1] 2 (3 / 2) => 1
// [4, 3, 2, 1] 2 (4 / 2) => 2
// [5, 4, 3, 2, 1] 3 (5 / 2) => 2
const size_t majority_index = confirmed_log_sizes.size() / 2;
const LogSize new_committed_log_size = confirmed_log_sizes[majority_index];
// We never go backwards in history.
MG_ASSERT(state_.committed_log_size <= new_committed_log_size,
"as a Leader, we have previously set our committed_log_size to {}, but our Followers have a majority "
"committed_log_size of {}",
state_.committed_log_size, new_committed_log_size);
state_.committed_log_size = new_committed_log_size;
// For each size between the old size and the new one (inclusive),
// Apply that log's WriteOperation to our replicated_state_,
// and use the specific return value of the ReplicatedState::Apply
// method (WriteResponseValue) to respond to the requester.
for (; state_.applied_size < state_.committed_log_size; state_.applied_size++) {
const LogIndex apply_index = state_.applied_size;
const auto &write_request = state_.log[apply_index].second;
const WriteResponseValue write_return = replicated_state_.Apply(write_request);
if (leader.pending_client_requests.contains(apply_index)) {
const PendingClientRequest client_request = std::move(leader.pending_client_requests.at(apply_index));
leader.pending_client_requests.erase(apply_index);
const WriteResponse<WriteResponseValue> resp{
.success = true,
.write_return = std::move(write_return),
.raft_index = apply_index,
};
io_.Send(client_request.address, client_request.request_id, std::move(resp));
}
}
Log("committed_log_size is now ", state_.committed_log_size);
}
// Raft paper - 5.1
// AppendEntries RPCs are initiated by leaders to replicate log entries and to provide a form of heartbeat
void BroadcastAppendEntries(std::map<Address, FollowerTracker> &followers) {
for (auto &[address, follower] : followers) {
const LogIndex next_index = follower.next_index;
const auto missing = state_.log.size() - next_index;
const auto batch_size = std::min(missing, kMaximumAppendBatchSize);
const auto start_index = next_index;
const auto end_index = start_index + batch_size;
// advance follower's next index
follower.next_index += batch_size;
std::vector<std::pair<Term, WriteOperation>> entries;
entries.insert(entries.begin(), state_.log.begin() + start_index, state_.log.begin() + end_index);
const Term previous_term_from_index = PreviousTermFromIndex(start_index);
Log("sending ", entries.size(), " entries to Follower ", address.last_known_port,
" which are above its next_index of ", next_index);
AppendRequest<WriteOperation> ar{
.term = state_.term,
.batch_start_log_index = start_index,
.last_log_term = previous_term_from_index,
.entries = std::move(entries),
.leader_commit = state_.committed_log_size,
};
// request_id not necessary to set because it's not a Future-backed Request.
static constexpr RequestId request_id = 0;
io_.Send(address, request_id, std::move(ar));
}
}
// Raft paper - 5.2
// Raft uses randomized election timeouts to ensure that split votes are rare and that they are resolved quickly
Duration RandomTimeout(Duration min, Duration max) {
std::uniform_int_distribution time_distrib(min.count(), max.count());
const auto rand_micros = io_.Rand(time_distrib);
return Duration{rand_micros};
}
Duration RandomTimeout(int min_micros, int max_micros) {
std::uniform_int_distribution time_distrib(min_micros, max_micros);
const int rand_micros = io_.Rand(time_distrib);
return std::chrono::microseconds{rand_micros};
}
Term PreviousTermFromIndex(LogIndex index) const {
if (index == 0 || state_.log.size() + 1 <= index) {
return 0;
}
const auto &[term, data] = state_.log.at(index - 1);
return term;
}
Term CommittedLogTerm() {
MG_ASSERT(state_.log.size() >= state_.committed_log_size);
if (state_.log.empty() || state_.committed_log_size == 0) {
return 0;
}
const auto &[term, data] = state_.log.at(state_.committed_log_size - 1);
return term;
}
Term LastLogTerm() const {
if (state_.log.empty()) {
return 0;
}
const auto &[term, data] = state_.log.back();
return term;
}
template <typename... Ts>
void Log(Ts &&...args) {
const Time now = io_.Now();
const auto micros = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
const Term term = state_.term;
const std::string role_string = std::visit([&](const auto &role) { return role.ToString(); }, role_);
std::ostringstream out;
out << '\t' << static_cast<int>(micros) << "\t" << term << "\t" << io_.GetAddress().last_known_port;
out << role_string;
(out << ... << args);
spdlog::info(out.str());
}
/////////////////////////////////////////////////////////////
/// Raft-related Cron methods
///
/// Cron + std::visit is how events are dispatched
/// to certain code based on Raft role.
///
/// Cron(role) takes as the first argument a reference to its
/// role, and as the second argument, the message that has
/// been received.
/////////////////////////////////////////////////////////////
// Raft paper - 5.2
// Candidates keep sending Vote to peers until:
// 1. receiving Append with a higher term (become Follower)
// 2. receiving Vote with a higher term (become a Follower)
// 3. receiving a quorum of responses to our last batch of Vote (become a Leader)
std::optional<Role> Cron(Candidate &candidate) {
const auto now = io_.Now();
const Duration election_timeout = RandomTimeout(kMinimumElectionTimeout, kMaximumElectionTimeout);
const auto election_timeout_us = std::chrono::duration_cast<std::chrono::milliseconds>(election_timeout).count();
if (now - candidate.election_began > election_timeout) {
state_.term++;
Log("becoming Candidate for term ", state_.term, " after leader timeout of ", election_timeout_us,
"ms elapsed since last election attempt");
const VoteRequest request{
.term = state_.term,
.log_size = state_.log.size(),
.last_log_term = LastLogTerm(),
};
auto outstanding_votes = std::set<Address>();
for (const auto &peer : peers_) {
// request_id not necessary to set because it's not a Future-backed Request.
static constexpr auto request_id = 0;
io_.template Send<VoteRequest>(peer, request_id, request);
outstanding_votes.insert(peer);
}
return Candidate{
.successful_votes = std::map<Address, LogIndex>(),
.election_began = now,
.outstanding_votes = outstanding_votes,
};
}
return std::nullopt;
}
// Raft paper - 5.2
// Followers become candidates if we haven't heard from the leader
// after a randomized timeout.
std::optional<Role> Cron(Follower &follower) {
const auto now = io_.Now();
const auto time_since_last_append_entries = now - follower.last_received_append_entries_timestamp;
// randomized follower timeout
const Duration election_timeout = RandomTimeout(kMinimumElectionTimeout, kMaximumElectionTimeout);
if (time_since_last_append_entries > election_timeout) {
// become a Candidate if we haven't heard from the Leader after this timeout
return Candidate{};
}
return std::nullopt;
}
// Leaders (re)send AppendRequest to followers.
std::optional<Role> Cron(Leader &leader) {
const Time now = io_.Now();
const Duration broadcast_timeout = RandomTimeout(kMinimumBroadcastTimeout, kMaximumBroadcastTimeout);
if (now - leader.last_broadcast > broadcast_timeout) {
BroadcastAppendEntries(leader.followers);
leader.last_broadcast = now;
}
return std::nullopt;
}
/////////////////////////////////////////////////////////////
/// Raft-related Handle methods
///
/// Handle + std::visit is how events are dispatched
/// to certain code based on Raft role.
///
/// Handle(role, message, ...)
/// takes as the first argument a reference
/// to its role, and as the second argument, the
/// message that has been received.
/////////////////////////////////////////////////////////////
// all roles can receive Vote and possibly become a follower
template <AllRoles ALL>
std::optional<Role> Handle(ALL & /* variable */, VoteRequest &&req, RequestId request_id, Address from_address) {
Log("received VoteRequest from ", from_address.last_known_port, " with term ", req.term);
const bool last_log_term_dominates = req.last_log_term >= LastLogTerm();
const bool term_dominates = req.term > state_.term;
const bool log_size_dominates = req.log_size >= state_.log.size();
const bool new_leader = last_log_term_dominates && term_dominates && log_size_dominates;
if (new_leader) {
MG_ASSERT(req.term > state_.term);
MG_ASSERT(std::max(req.term, state_.term) == req.term);
}
const VoteResponse res{
.term = std::max(req.term, state_.term),
.committed_log_size = state_.committed_log_size,
.vote_granted = new_leader,
};
io_.Send(from_address, request_id, res);
if (new_leader) {
// become a follower
state_.term = req.term;
return Follower{
.last_received_append_entries_timestamp = io_.Now(),
.leader_address = from_address,
};
}
if (term_dominates) {
Log("received a vote from an inferior candidate. Becoming Candidate");
state_.term = std::max(state_.term, req.term) + 1;
return Candidate{};
}
return std::nullopt;
}
std::optional<Role> Handle(Candidate &candidate, VoteResponse &&res, RequestId /* variable */, Address from_address) {
Log("received VoteResponse");
if (!res.vote_granted || res.term != state_.term) {
Log("received unsuccessful VoteResponse from term ", res.term, " when our candidacy term is ", state_.term);
// we received a delayed VoteResponse from the past, which has to do with an election that is
// no longer valid. We can simply drop this.
return std::nullopt;
}
MG_ASSERT(candidate.outstanding_votes.contains(from_address),
"Received unexpected VoteResponse from server not present in Candidate's outstanding_votes!");
candidate.outstanding_votes.erase(from_address);
MG_ASSERT(!candidate.successful_votes.contains(from_address),
"Received unexpected VoteResponse from server already in Candidate's successful_votes!");
candidate.successful_votes.insert({from_address, res.committed_log_size});
if (candidate.successful_votes.size() >= candidate.outstanding_votes.size()) {
std::map<Address, FollowerTracker> followers{};
for (const auto &[address, committed_log_size] : candidate.successful_votes) {
FollowerTracker follower{
.next_index = committed_log_size,
.confirmed_log_size = committed_log_size,
};
followers.insert({address, follower});
}
for (const auto &address : candidate.outstanding_votes) {
FollowerTracker follower{
.next_index = state_.log.size(),
.confirmed_log_size = 0,
};
followers.insert({address, follower});
}
Log("becoming Leader at term ", state_.term);
BroadcastAppendEntries(followers);
return Leader{
.followers = std::move(followers),
.pending_client_requests = std::unordered_map<LogIndex, PendingClientRequest>(),
};
}
return std::nullopt;
}
template <LeaderOrFollower LOF>
std::optional<Role> Handle(LOF & /* variable */, VoteResponse && /* variable */, RequestId /* variable */,
Address /* variable */) {
Log("non-Candidate received VoteResponse");
return std::nullopt;
}
template <AllRoles ALL>
std::optional<Role> Handle(ALL &role, AppendRequest<WriteOperation> &&req, RequestId request_id,
Address from_address) {
// log size starts out as state_.committed_log_size and only if everything is successful do we
// switch it to the log length.
AppendResponse res{
.success = false,
.term = state_.term,
.last_log_term = CommittedLogTerm(),
.log_size = state_.log.size(),
};
if constexpr (std::is_same<ALL, Leader>()) {
MG_ASSERT(req.term != state_.term, "Multiple leaders are acting under the term ", req.term);
}
const bool is_candidate = std::is_same<ALL, Candidate>();
const bool is_failed_competitor = is_candidate && req.term == state_.term;
const Time now = io_.Now();
// Raft paper - 5.2
// While waiting for votes, a candidate may receive an
// AppendEntries RPC from another server claiming to be leader. If
// the leaders term (included in its RPC) is at least as large as
// the candidates current term, then the candidate recognizes the
// leader as legitimate and returns to follower state.
if (req.term > state_.term || is_failed_competitor) {
// become follower of this leader, reply with our log status
state_.term = req.term;
io_.Send(from_address, request_id, res);
Log("becoming Follower of Leader ", from_address.last_known_port, " at term ", req.term);
return Follower{
.last_received_append_entries_timestamp = now,
.leader_address = from_address,
};
}
if (req.term < state_.term) {
// nack this request from an old leader
io_.Send(from_address, request_id, res);
return std::nullopt;
}
// at this point, we're dealing with our own leader
if constexpr (std::is_same<ALL, Follower>()) {
// small specialization for when we're already a Follower
MG_ASSERT(role.leader_address == from_address, "Multiple Leaders are acting under the same term number!");
role.last_received_append_entries_timestamp = now;
} else {
Log("Somehow entered Follower-specific logic as a non-Follower");
MG_ASSERT(false, "Somehow entered Follower-specific logic as a non-Follower");
}
// Handle steady-state conditions.
if (req.batch_start_log_index != state_.log.size()) {
Log("req.batch_start_log_index of ", req.batch_start_log_index, " does not match our log size of ",
state_.log.size());
} else if (req.last_log_term != LastLogTerm()) {
Log("req.last_log_term differs from our leader term at that slot, expected: ", LastLogTerm(), " but got ",
req.last_log_term);
} else {
// happy path - Apply log
Log("applying batch of ", req.entries.size(), " entries to our log starting at index ",
req.batch_start_log_index);
const auto resize_length = req.batch_start_log_index;
MG_ASSERT(resize_length >= state_.committed_log_size,
"Applied history from Leader which goes back in time from our commit_index");
// possibly chop-off stuff that was replaced by
// things with different terms (we got data that
// hasn't reached consensus yet, which is normal)
state_.log.resize(resize_length);
if (req.entries.size() > 0) {
auto &[first_term, op] = req.entries.at(0);
MG_ASSERT(LastLogTerm() <= first_term);
}
state_.log.insert(state_.log.end(), std::make_move_iterator(req.entries.begin()),
std::make_move_iterator(req.entries.end()));
MG_ASSERT(req.leader_commit >= state_.committed_log_size);
state_.committed_log_size = std::min(req.leader_commit, state_.log.size());
for (; state_.applied_size < state_.committed_log_size; state_.applied_size++) {
const auto &write_request = state_.log[state_.applied_size].second;
replicated_state_.Apply(write_request);
}
res.success = true;
}
res.last_log_term = LastLogTerm();
res.log_size = state_.log.size();
Log("returning log_size of ", res.log_size);
io_.Send(from_address, request_id, res);
return std::nullopt;
}
std::optional<Role> Handle(Leader &leader, AppendResponse &&res, RequestId /* variable */, Address from_address) {
if (res.term != state_.term) {
Log("received AppendResponse related to a previous term when we (presumably) were the leader");
return std::nullopt;
}
// TODO(tyler) when we have dynamic membership, this assert will become incorrect, but we should
// keep it in-place until then because it has bug finding value.
MG_ASSERT(leader.followers.contains(from_address), "received AppendResponse from unknown Follower");
// at this point, we know the term matches and we know this Follower
FollowerTracker &follower = leader.followers.at(from_address);
if (res.success) {
Log("got successful AppendResponse from ", from_address.last_known_port, " with log_size of ", res.log_size);
follower.next_index = std::max(follower.next_index, res.log_size);
} else {
Log("got unsuccessful AppendResponse from ", from_address.last_known_port, " with log_size of ", res.log_size);
follower.next_index = res.log_size;
}
follower.confirmed_log_size = std::max(follower.confirmed_log_size, res.log_size);
BumpCommitIndexAndReplyToClients(leader);
return std::nullopt;
}
template <FollowerOrCandidate FOC>
std::optional<Role> Handle(FOC & /* variable */, AppendResponse && /* variable */, RequestId /* variable */,
Address /* variable */) {
// we used to be the leader, and are getting old delayed responses
return std::nullopt;
}
/////////////////////////////////////////////////////////////
/// RSM-related handle methods
/////////////////////////////////////////////////////////////
// Leaders are able to immediately respond to the requester (with a ReadResponseValue) applied to the ReplicatedState
std::optional<Role> Handle(Leader & /* variable */, ReadRequest<ReadOperation> &&req, RequestId request_id,
Address from_address) {
Log("handling ReadOperation");
ReadOperation read_operation = req.operation;
ReadResponseValue read_return = replicated_state_.Read(read_operation);
const ReadResponse<ReadResponseValue> resp{
.success = true,
.read_return = std::move(read_return),
.retry_leader = std::nullopt,
};
io_.Send(from_address, request_id, resp);
return std::nullopt;
}
// Candidates should respond with a failure, similar to the Candidate + WriteRequest failure below
std::optional<Role> Handle(Candidate & /* variable */, ReadRequest<ReadOperation> && /* variable */,
RequestId request_id, Address from_address) {
Log("received ReadOperation - not redirecting because no Leader is known");
const ReadResponse<ReadResponseValue> res{
.success = false,
};
io_.Send(from_address, request_id, res);
Cron();
return std::nullopt;
}
// Followers should respond with a redirection, similar to the Follower + WriteRequest response below
std::optional<Role> Handle(Follower &follower, ReadRequest<ReadOperation> && /* variable */, RequestId request_id,
Address from_address) {
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
const ReadResponse<ReadResponseValue> res{
.success = false,
.retry_leader = follower.leader_address,
};
io_.Send(from_address, request_id, res);
return std::nullopt;
}
// Raft paper - 8
// When a client first starts up, it connects to a randomly chosen
// server. If the clients first choice is not the leader, that
// server will reject the clients request and supply information
// about the most recent leader it has heard from.
std::optional<Role> Handle(Follower &follower, WriteRequest<WriteOperation> && /* variable */, RequestId request_id,
Address from_address) {
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
const WriteResponse<WriteResponseValue> res{
.success = false,
.retry_leader = follower.leader_address,
};
io_.Send(from_address, request_id, res);
return std::nullopt;
}
std::optional<Role> Handle(Candidate & /* variable */, WriteRequest<WriteOperation> && /* variable */,
RequestId request_id, Address from_address) {
Log("received WriteRequest - not redirecting because no Leader is known");
const WriteResponse<WriteResponseValue> res{
.success = false,
};
io_.Send(from_address, request_id, res);
Cron();
return std::nullopt;
}
// only leaders actually handle replication requests from clients
std::optional<Role> Handle(Leader &leader, WriteRequest<WriteOperation> &&req, RequestId request_id,
Address from_address) {
Log("handling WriteRequest");
// we are the leader. add item to log and send Append to peers
MG_ASSERT(state_.term >= LastLogTerm());
state_.log.emplace_back(std::pair(state_.term, std::move(req.operation)));
LogIndex log_index = state_.log.size() - 1;
PendingClientRequest pcr{
.request_id = request_id,
.address = from_address,
.received_at = io_.Now(),
};
leader.pending_client_requests.emplace(log_index, pcr);
if (peers_.empty()) {
BumpCommitIndexAndReplyToClients(leader);
} else {
BroadcastAppendEntries(leader.followers);
}
return std::nullopt;
}
};
}; // namespace memgraph::io::rsm

136
src/io/rsm/rsm_client.hpp Normal file
View File

@ -0,0 +1,136 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <iostream>
#include <optional>
#include <vector>
#include "io/address.hpp"
#include "io/rsm/raft.hpp"
#include "utils/result.hpp"
namespace memgraph::io::rsm {
using memgraph::io::Address;
using memgraph::io::Duration;
using memgraph::io::ResponseEnvelope;
using memgraph::io::ResponseFuture;
using memgraph::io::ResponseResult;
using memgraph::io::Time;
using memgraph::io::TimedOut;
using memgraph::io::rsm::ReadRequest;
using memgraph::io::rsm::ReadResponse;
using memgraph::io::rsm::WriteRequest;
using memgraph::io::rsm::WriteResponse;
using memgraph::utils::BasicResult;
template <typename IoImpl, typename WriteRequestT, typename WriteResponseT, typename ReadRequestT,
typename ReadResponseT>
class RsmClient {
using ServerPool = std::vector<Address>;
Io<IoImpl> io_;
Address leader_;
ServerPool server_addrs_;
template <typename ResponseT>
void PossiblyRedirectLeader(const ResponseT &response) {
if (response.retry_leader) {
MG_ASSERT(!response.success, "retry_leader should never be set for successful responses");
leader_ = response.retry_leader.value();
spdlog::debug("client redirected to leader server {}", leader_.ToString());
} else if (!response.success) {
std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
size_t addr_index = io_.Rand(addr_distrib);
leader_ = server_addrs_[addr_index];
spdlog::debug(
"client NOT redirected to leader server despite our success failing to be processed (it probably was sent to "
"a RSM Candidate) trying a random one at index {} with address {}",
addr_index, leader_.ToString());
}
}
public:
RsmClient(Io<IoImpl> io, Address leader, ServerPool server_addrs)
: io_{io}, leader_{leader}, server_addrs_{server_addrs} {}
RsmClient() = delete;
BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
WriteRequest<WriteRequestT> client_req;
client_req.operation = req;
const Duration overall_timeout = io_.GetDefaultTimeout();
const Time before = io_.Now();
do {
spdlog::debug("client sending WriteRequest to Leader {}", leader_.ToString());
ResponseFuture<WriteResponse<WriteResponseT>> response_future =
io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, client_req);
ResponseResult<WriteResponse<WriteResponseT>> response_result = std::move(response_future).Wait();
if (response_result.HasError()) {
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
return response_result.GetError();
}
ResponseEnvelope<WriteResponse<WriteResponseT>> &&response_envelope = std::move(response_result.GetValue());
WriteResponse<WriteResponseT> &&write_response = std::move(response_envelope.message);
if (write_response.success) {
return std::move(write_response.write_return);
}
PossiblyRedirectLeader(write_response);
} while (io_.Now() < before + overall_timeout);
return TimedOut{};
}
BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
ReadRequest<ReadRequestT> read_req;
read_req.operation = req;
const Duration overall_timeout = io_.GetDefaultTimeout();
const Time before = io_.Now();
do {
spdlog::debug("client sending ReadRequest to Leader {}", leader_.ToString());
ResponseFuture<ReadResponse<ReadResponseT>> get_response_future =
io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
// receive response
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(get_response_future).Wait();
if (get_response_result.HasError()) {
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
return get_response_result.GetError();
}
ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
if (read_get_response.success) {
return std::move(read_get_response.read_return);
}
PossiblyRedirectLeader(read_get_response);
} while (io_.Now() < before + overall_timeout);
return TimedOut{};
}
};
} // namespace memgraph::io::rsm

155
src/io/rsm/shard_rsm.hpp Normal file
View File

@ -0,0 +1,155 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
/// The ShardRsm is a simple in-memory raft-backed kv store that can be used for simple testing
/// and implementation of some query engine logic before storage engines are fully implemented.
///
/// To implement multiple read and write commands, change the StorageRead* and StorageWrite* requests
/// and responses to a std::variant of the different options, and route them to specific handlers in
/// the ShardRsm's Read and Apply methods. Remember that Read is called immediately when the Raft
/// leader receives the request, and does not replicate anything over Raft. Apply is called only
/// AFTER the StorageWriteRequest is replicated to a majority of Raft peers, and the result of calling
/// ShardRsm::Apply(StorageWriteRequest) is returned to the client that submitted the request.
#include <deque>
#include <iostream>
#include <map>
#include <optional>
#include <set>
#include <thread>
#include <vector>
#include "coordinator/hybrid_logical_clock.hpp"
#include "io/address.hpp"
#include "io/rsm/raft.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "utils/logging.hpp"
namespace memgraph::io::rsm {
using memgraph::coordinator::Hlc;
using memgraph::storage::v3::LabelId;
using memgraph::storage::v3::PropertyValue;
using ShardRsmKey = std::vector<PropertyValue>;
struct StorageWriteRequest {
LabelId label_id;
Hlc transaction_id;
ShardRsmKey key;
std::optional<int> value;
};
struct StorageWriteResponse {
bool shard_rsm_success;
std::optional<int> last_value;
// Only has a value if the given shard does not contain the requested key
std::optional<Hlc> latest_known_shard_map_version{std::nullopt};
};
struct StorageReadRequest {
LabelId label_id;
Hlc transaction_id;
ShardRsmKey key;
};
struct StorageReadResponse {
bool shard_rsm_success;
std::optional<int> value;
// Only has a value if the given shard does not contain the requested key
std::optional<Hlc> latest_known_shard_map_version{std::nullopt};
};
class ShardRsm {
std::map<ShardRsmKey, int> state_;
ShardRsmKey minimum_key_;
std::optional<ShardRsmKey> maximum_key_{std::nullopt};
Hlc shard_map_version_;
// The key is not located in this shard
bool IsKeyInRange(const ShardRsmKey &key) const {
if (maximum_key_) [[likely]] {
return (key >= minimum_key_ && key <= maximum_key_);
}
return key >= minimum_key_;
}
public:
StorageReadResponse Read(StorageReadRequest request) const {
StorageReadResponse ret;
if (!IsKeyInRange(request.key)) {
ret.latest_known_shard_map_version = shard_map_version_;
ret.shard_rsm_success = false;
} else if (state_.contains(request.key)) {
ret.value = state_.at(request.key);
ret.shard_rsm_success = true;
} else {
ret.shard_rsm_success = false;
ret.value = std::nullopt;
}
return ret;
}
StorageWriteResponse Apply(StorageWriteRequest request) {
StorageWriteResponse ret;
// Key is outside the prohibited range
if (!IsKeyInRange(request.key)) {
ret.latest_known_shard_map_version = shard_map_version_;
ret.shard_rsm_success = false;
}
// Key exist
else if (state_.contains(request.key)) {
auto &val = state_[request.key];
/*
* Delete
*/
if (!request.value) {
ret.shard_rsm_success = true;
ret.last_value = val;
state_.erase(state_.find(request.key));
}
/*
* Update
*/
// Does old_value match?
if (request.value == val) {
ret.last_value = val;
ret.shard_rsm_success = true;
val = request.value.value();
} else {
ret.last_value = val;
ret.shard_rsm_success = false;
}
}
/*
* Create
*/
else {
ret.last_value = std::nullopt;
ret.shard_rsm_success = true;
state_.emplace(request.key, request.value.value());
}
return ret;
}
};
} // namespace memgraph::io::rsm

View File

@ -0,0 +1,8 @@
set(io_simulator_sources
simulator_handle.cpp)
find_package(fmt REQUIRED)
find_package(Threads REQUIRED)
add_library(mg-io-simulator STATIC ${io_simulator_sources})
target_link_libraries(mg-io-simulator stdc++fs Threads::Threads fmt::fmt mg-utils)

View File

@ -0,0 +1,51 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <memory>
#include <random>
#include "io/address.hpp"
#include "io/simulator/simulator_config.hpp"
#include "io/simulator/simulator_handle.hpp"
#include "io/simulator/simulator_transport.hpp"
namespace memgraph::io::simulator {
class Simulator {
std::mt19937 rng_;
std::shared_ptr<SimulatorHandle> simulator_handle_;
uint16_t auto_port_ = 0;
public:
explicit Simulator(SimulatorConfig config)
: rng_(std::mt19937{config.rng_seed}), simulator_handle_{std::make_shared<SimulatorHandle>(config)} {}
void ShutDown() { simulator_handle_->ShutDown(); }
Io<SimulatorTransport> RegisterNew() {
Address address = Address::TestAddress(auto_port_++);
return Register(address);
}
Io<SimulatorTransport> Register(Address address) {
std::uniform_int_distribution<uint64_t> seed_distrib;
uint64_t seed = seed_distrib(rng_);
return Io{SimulatorTransport{simulator_handle_, address, seed}, address};
}
void IncrementServerCountAndWaitForQuiescentState(Address address) {
simulator_handle_->IncrementServerCountAndWaitForQuiescentState(address);
}
SimulatorStats Stats() { return simulator_handle_->Stats(); }
};
}; // namespace memgraph::io::simulator

View File

@ -0,0 +1,30 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include "io/time.hpp"
namespace memgraph::io::simulator {
using memgraph::io::Time;
struct SimulatorConfig {
uint8_t drop_percent = 0;
bool perform_timeouts = false;
bool scramble_messages = true;
uint64_t rng_seed = 0;
Time start_time = Time::min();
Time abort_time = Time::max();
};
}; // namespace memgraph::io::simulator

View File

@ -0,0 +1,142 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "io/simulator/simulator_handle.hpp"
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/simulator/simulator_config.hpp"
#include "io/simulator/simulator_stats.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
namespace memgraph::io::simulator {
using memgraph::io::Duration;
using memgraph::io::Time;
void SimulatorHandle::ShutDown() {
std::unique_lock<std::mutex> lock(mu_);
should_shut_down_ = true;
cv_.notify_all();
}
bool SimulatorHandle::ShouldShutDown() const {
std::unique_lock<std::mutex> lock(mu_);
return should_shut_down_;
}
void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address address) {
std::unique_lock<std::mutex> lock(mu_);
server_addresses_.insert(address);
while (true) {
const size_t blocked_servers = blocked_on_receive_.size();
const bool all_servers_blocked = blocked_servers == server_addresses_.size();
if (all_servers_blocked) {
return;
}
cv_.wait(lock);
}
}
bool SimulatorHandle::MaybeTickSimulator() {
std::unique_lock<std::mutex> lock(mu_);
const size_t blocked_servers = blocked_on_receive_.size();
if (blocked_servers < server_addresses_.size()) {
// we only need to advance the simulator when all
// servers have reached a quiescent state, blocked
// on their own futures or receive methods.
return false;
}
stats_.simulator_ticks++;
cv_.notify_all();
TimeoutPromisesPastDeadline();
if (in_flight_.empty()) {
// return early here because there are no messages to schedule
// We tick the clock forward when all servers are blocked but
// there are no in-flight messages to schedule delivery of.
std::poisson_distribution<> time_distrib(50);
Duration clock_advance = std::chrono::microseconds{time_distrib(rng_)};
cluster_wide_time_microseconds_ += clock_advance;
MG_ASSERT(cluster_wide_time_microseconds_ < config_.abort_time,
"Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
"in an expected amount of time.");
return true;
}
if (config_.scramble_messages) {
// scramble messages
std::uniform_int_distribution<size_t> swap_distrib(0, in_flight_.size() - 1);
const size_t swap_index = swap_distrib(rng_);
std::swap(in_flight_[swap_index], in_flight_.back());
}
auto [to_address, opaque_message] = std::move(in_flight_.back());
in_flight_.pop_back();
std::uniform_int_distribution<int> drop_distrib(0, 99);
const int drop_threshold = drop_distrib(rng_);
const bool should_drop = drop_threshold < config_.drop_percent;
if (should_drop) {
stats_.dropped_messages++;
}
PromiseKey promise_key{.requester_address = to_address,
.request_id = opaque_message.request_id,
.replier_address = opaque_message.from_address};
if (promises_.contains(promise_key)) {
// complete waiting promise if it's there
DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
promises_.erase(promise_key);
const bool normal_timeout = config_.perform_timeouts && (dop.deadline < cluster_wide_time_microseconds_);
if (should_drop || normal_timeout) {
stats_.timed_out_requests++;
dop.promise.TimeOut();
} else {
stats_.total_responses++;
dop.promise.Fill(std::move(opaque_message));
}
} else if (should_drop) {
// don't add it anywhere, let it drop
} else {
// add to can_receive_ if not
const auto &[om_vec, inserted] = can_receive_.try_emplace(to_address, std::vector<OpaqueMessage>());
om_vec->second.emplace_back(std::move(opaque_message));
}
return true;
}
Time SimulatorHandle::Now() const {
std::unique_lock<std::mutex> lock(mu_);
return cluster_wide_time_microseconds_;
}
SimulatorStats SimulatorHandle::Stats() {
std::unique_lock<std::mutex> lock(mu_);
return stats_;
}
} // namespace memgraph::io::simulator

View File

@ -0,0 +1,177 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <any>
#include <compare>
#include <iostream>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include <utility>
#include <variant>
#include <vector>
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/message_conversion.hpp"
#include "io/simulator/simulator_config.hpp"
#include "io/simulator/simulator_stats.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
namespace memgraph::io::simulator {
class SimulatorHandle {
mutable std::mutex mu_{};
mutable std::condition_variable cv_;
// messages that have not yet been scheduled or dropped
std::vector<std::pair<Address, OpaqueMessage>> in_flight_;
// the responses to requests that are being waited on
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
// messages that are sent to servers that may later receive them
std::map<Address, std::vector<OpaqueMessage>> can_receive_;
Time cluster_wide_time_microseconds_;
bool should_shut_down_ = false;
SimulatorStats stats_;
std::set<Address> blocked_on_receive_;
std::set<Address> server_addresses_;
std::mt19937 rng_;
SimulatorConfig config_;
void TimeoutPromisesPastDeadline() {
const Time now = cluster_wide_time_microseconds_;
for (auto it = promises_.begin(); it != promises_.end();) {
auto &[promise_key, dop] = *it;
if (dop.deadline < now) {
spdlog::info("timing out request from requester {} to replier {}.", promise_key.requester_address.ToString(),
promise_key.replier_address.ToString());
std::move(dop).promise.TimeOut();
it = promises_.erase(it);
stats_.timed_out_requests++;
} else {
++it;
}
}
}
public:
explicit SimulatorHandle(SimulatorConfig config)
: cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {}
void IncrementServerCountAndWaitForQuiescentState(Address address);
/// This method causes most of the interesting simulation logic to happen, wrt network behavior.
/// It checks to see if all background "server" threads are blocked on new messages, and if so,
/// it will decide whether to drop, reorder, or deliver in-flight messages based on the SimulatorConfig
/// that was used to create the Simulator.
bool MaybeTickSimulator();
void ShutDown();
bool ShouldShutDown() const;
template <Message Request, Message Response>
void SubmitRequest(Address to_address, Address from_address, RequestId request_id, Request &&request,
Duration timeout, ResponsePromise<Response> &&promise) {
std::unique_lock<std::mutex> lock(mu_);
const Time deadline = cluster_wide_time_microseconds_ + timeout;
std::any message(request);
OpaqueMessage om{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message)};
in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
OpaquePromise opaque_promise(std::move(promise).ToUnique());
DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
promises_.emplace(std::move(promise_key), std::move(dop));
stats_.total_messages++;
stats_.total_requests++;
cv_.notify_all();
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(const Address &receiver, Duration timeout) {
std::unique_lock<std::mutex> lock(mu_);
blocked_on_receive_.emplace(receiver);
const Time deadline = cluster_wide_time_microseconds_ + timeout;
while (!should_shut_down_ && (cluster_wide_time_microseconds_ < deadline)) {
if (can_receive_.contains(receiver)) {
std::vector<OpaqueMessage> &can_rx = can_receive_.at(receiver);
if (!can_rx.empty()) {
OpaqueMessage message = std::move(can_rx.back());
can_rx.pop_back();
// TODO(tyler) search for item in can_receive_ that matches the desired types, rather
// than asserting that the last item in can_rx matches.
auto m_opt = std::move(message).Take<Ms...>();
blocked_on_receive_.erase(receiver);
return std::move(m_opt).value();
}
}
lock.unlock();
bool made_progress = MaybeTickSimulator();
lock.lock();
if (!should_shut_down_ && !made_progress) {
cv_.wait(lock);
}
}
blocked_on_receive_.erase(receiver);
return TimedOut{};
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, M message) {
std::unique_lock<std::mutex> lock(mu_);
std::any message_any(std::move(message));
OpaqueMessage om{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message_any)};
in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
stats_.total_messages++;
cv_.notify_all();
}
Time Now() const;
template <class D = std::poisson_distribution<>, class Return = uint64_t>
Return Rand(D distrib) {
std::unique_lock<std::mutex> lock(mu_);
return distrib(rng_);
}
SimulatorStats Stats();
};
}; // namespace memgraph::io::simulator

View File

@ -0,0 +1,25 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cstdint>
namespace memgraph::io::simulator {
struct SimulatorStats {
uint64_t total_messages = 0;
uint64_t dropped_messages = 0;
uint64_t timed_out_requests = 0;
uint64_t total_requests = 0;
uint64_t total_responses = 0;
uint64_t simulator_ticks = 0;
};
}; // namespace memgraph::io::simulator

View File

@ -0,0 +1,67 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <memory>
#include <utility>
#include "io/address.hpp"
#include "io/simulator/simulator_handle.hpp"
#include "io/time.hpp"
namespace memgraph::io::simulator {
using memgraph::io::Duration;
using memgraph::io::Time;
class SimulatorTransport {
std::shared_ptr<SimulatorHandle> simulator_handle_;
const Address address_;
std::mt19937 rng_;
public:
SimulatorTransport(std::shared_ptr<SimulatorHandle> simulator_handle, Address address, uint64_t seed)
: simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, uint64_t request_id, RequestT request,
Duration timeout) {
std::function<bool()> maybe_tick_simulator = [this] { return simulator_handle_->MaybeTickSimulator(); };
auto [future, promise] =
memgraph::io::FuturePromisePairWithNotifier<ResponseResult<ResponseT>>(maybe_tick_simulator);
simulator_handle_->SubmitRequest(to_address, from_address, request_id, std::move(request), timeout,
std::move(promise));
return std::move(future);
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address receiver_address, Duration timeout) {
return simulator_handle_->template Receive<Ms...>(receiver_address, timeout);
}
template <Message M>
void Send(Address to_address, Address from_address, uint64_t request_id, M message) {
return simulator_handle_->template Send<M>(to_address, from_address, request_id, message);
}
Time Now() const { return simulator_handle_->Now(); }
bool ShouldShutDown() const { return simulator_handle_->ShouldShutDown(); }
template <class D = std::poisson_distribution<>, class Return = uint64_t>
Return Rand(D distrib) {
return distrib(rng_);
}
};
}; // namespace memgraph::io::simulator

21
src/io/time.hpp Normal file
View File

@ -0,0 +1,21 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
namespace memgraph::io {
using Duration = std::chrono::microseconds;
using Time = std::chrono::time_point<std::chrono::system_clock, Duration>;
} // namespace memgraph::io

144
src/io/transport.hpp Normal file
View File

@ -0,0 +1,144 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <concepts>
#include <random>
#include <variant>
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/future.hpp"
#include "io/time.hpp"
#include "utils/result.hpp"
namespace memgraph::io {
using memgraph::utils::BasicResult;
// TODO(tyler) ensure that Message continues to represent
// reasonable constraints around message types over time,
// as we adapt things to use Thrift-generated message types.
template <typename T>
concept Message = std::same_as<T, std::decay_t<T>>;
using RequestId = uint64_t;
template <Message M>
struct ResponseEnvelope {
M message;
RequestId request_id;
Address to_address;
Address from_address;
};
template <Message M>
using ResponseResult = BasicResult<TimedOut, ResponseEnvelope<M>>;
template <Message M>
using ResponseFuture = memgraph::io::Future<ResponseResult<M>>;
template <Message M>
using ResponsePromise = memgraph::io::Promise<ResponseResult<M>>;
template <Message... Ms>
struct RequestEnvelope {
std::variant<Ms...> message;
RequestId request_id;
Address to_address;
Address from_address;
};
template <Message... Ms>
using RequestResult = BasicResult<TimedOut, RequestEnvelope<Ms...>>;
template <typename I>
class Io {
I implementation_;
Address address_;
RequestId request_id_counter_ = 0;
Duration default_timeout_ = std::chrono::microseconds{100000};
public:
Io(I io, Address address) : implementation_(io), address_(address) {}
/// Set the default timeout for all requests that are issued
/// without an explicit timeout set.
void SetDefaultTimeout(Duration timeout) { default_timeout_ = timeout; }
/// Returns the current default timeout for this Io instance.
Duration GetDefaultTimeout() { return default_timeout_; }
/// Issue a request with an explicit timeout in microseconds provided. This tends to be used by clients.
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> RequestWithTimeout(Address address, RequestT request, Duration timeout) {
const RequestId request_id = ++request_id_counter_;
const Address from_address = address_;
return implementation_.template Request<RequestT, ResponseT>(address, from_address, request_id, request, timeout);
}
/// Issue a request that times out after the default timeout. This tends
/// to be used by clients.
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, RequestT request) {
const RequestId request_id = ++request_id_counter_;
const Duration timeout = default_timeout_;
const Address from_address = address_;
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, request_id,
std::move(request), timeout);
}
/// Wait for an explicit number of microseconds for a request of one of the
/// provided types to arrive. This tends to be used by servers.
template <Message... Ms>
RequestResult<Ms...> ReceiveWithTimeout(Duration timeout) {
return implementation_.template Receive<Ms...>(address_, timeout);
}
/// Wait the default number of microseconds for a request of one of the
/// provided types to arrive. This tends to be used by servers.
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive() {
const Duration timeout = default_timeout_;
return implementation_.template Receive<Ms...>(address_, timeout);
}
/// Send a message in a best-effort fashion. This is used for messaging where
/// responses are not necessarily expected, and for servers to respond to requests.
/// If you need reliable delivery, this must be built on-top. TCP is not enough for most use cases.
template <Message M>
void Send(Address to_address, RequestId request_id, M message) {
Address from_address = address_;
return implementation_.template Send<M>(to_address, from_address, request_id, std::move(message));
}
/// The current system time. This time source should be preferred over any other,
/// because it lets us deterministically control clocks from tests for making
/// things like timeouts deterministic.
Time Now() const { return implementation_.Now(); }
/// Returns true if the system should shut-down.
bool ShouldShutDown() const { return implementation_.ShouldShutDown(); }
/// Returns a random number within the specified distribution.
template <class D = std::poisson_distribution<>, class Return = uint64_t>
Return Rand(D distrib) {
return implementation_.template Rand<D, Return>(distrib);
}
Address GetAddress() { return address_; }
void SetAddress(Address address) { address_ = address; }
Io<I> ForkLocal() { return Io(implementation_, address_.ForkUniqueAddress()); }
};
}; // namespace memgraph::io

View File

@ -0,0 +1,42 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <boost/asio/ip/tcp.hpp>
#include "io/address.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/schemas.hpp"
namespace memgraph::machine_manager {
using memgraph::io::Address;
using memgraph::storage::v3::SchemaProperty;
using CompoundKey = std::vector<memgraph::storage::v3::PropertyValue>;
struct InitialLabelSpace {
std::string label_name;
std::vector<SchemaProperty> schema;
size_t replication_factor;
std::vector<CompoundKey> split_points;
};
struct MachineConfig {
std::vector<InitialLabelSpace> initial_label_spaces;
std::vector<Address> coordinator_addresses;
bool is_storage;
bool is_coordinator;
bool is_query_engine;
boost::asio::ip::address listen_ip;
uint16_t listen_port;
};
} // namespace memgraph::machine_manager

View File

@ -0,0 +1,177 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <coordinator/coordinator_rsm.hpp>
#include <io/message_conversion.hpp>
#include <io/messages.hpp>
#include <io/rsm/rsm_client.hpp>
#include <io/time.hpp>
#include <machine_manager/machine_config.hpp>
#include <storage/v3/shard_manager.hpp>
namespace memgraph::machine_manager {
using memgraph::coordinator::Coordinator;
using memgraph::coordinator::CoordinatorReadRequests;
using memgraph::coordinator::CoordinatorReadResponses;
using memgraph::coordinator::CoordinatorRsm;
using memgraph::coordinator::CoordinatorWriteRequests;
using memgraph::coordinator::CoordinatorWriteResponses;
using memgraph::io::ConvertVariant;
using memgraph::io::Duration;
using memgraph::io::RequestId;
using memgraph::io::Time;
using memgraph::io::messages::CoordinatorMessages;
using memgraph::io::messages::ShardManagerMessages;
using memgraph::io::messages::ShardMessages;
using memgraph::io::messages::StorageReadRequest;
using memgraph::io::messages::StorageWriteRequest;
using memgraph::io::rsm::AppendRequest;
using memgraph::io::rsm::AppendResponse;
using memgraph::io::rsm::ReadRequest;
using memgraph::io::rsm::VoteRequest;
using memgraph::io::rsm::VoteResponse;
using memgraph::io::rsm::WriteRequest;
using memgraph::io::rsm::WriteResponse;
using memgraph::storage::v3::ShardManager;
/// The MachineManager is responsible for:
/// * starting the entire system and ensuring that high-level
/// operational requirements continue to be met
/// * acting as a machine's single caller of Io::Receive
/// * routing incoming messages from the Io interface to the
/// appropriate Coordinator or to the StorageManager
/// (it's not necessary to route anything in this layer
/// to the query engine because the query engine only
/// communicates using higher-level Futures that will
/// be filled immediately when the response comes in
/// at the lower-level transport layer.
///
/// Every storage engine has exactly one RsmEngine.
template <typename IoImpl>
class MachineManager {
io::Io<IoImpl> io_;
MachineConfig config_;
CoordinatorRsm<IoImpl> coordinator_;
ShardManager<IoImpl> shard_manager_;
Time next_cron_;
public:
// TODO initialize ShardManager with "real" coordinator addresses instead of io.GetAddress
// which is only true for single-machine config.
MachineManager(io::Io<IoImpl> io, MachineConfig config, Coordinator coordinator)
: io_(io),
config_(config),
coordinator_{std::move(io.ForkLocal()), {}, std::move(coordinator)},
shard_manager_(ShardManager{io.ForkLocal(), coordinator_.GetAddress()}) {}
Address CoordinatorAddress() { return coordinator_.GetAddress(); }
void Run() {
while (!io_.ShouldShutDown()) {
const auto now = io_.Now();
if (now >= next_cron_) {
next_cron_ = Cron();
}
Duration receive_timeout = next_cron_ - now;
// Note: this parameter pack must be kept in-sync with the ReceiveWithTimeout parameter pack below
using AllMessages =
std::variant<ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>, AppendResponse,
WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse,
WriteResponse<CoordinatorWriteResponses>, ReadRequest<StorageReadRequest>,
AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>>;
spdlog::info("MM waiting on Receive");
// Note: this parameter pack must be kept in-sync with the AllMessages parameter pack above
auto request_result = io_.template ReceiveWithTimeout<
ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>, AppendResponse,
WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse, WriteResponse<CoordinatorWriteResponses>,
ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>>(
receive_timeout);
if (request_result.HasError()) {
// time to do Cron
spdlog::info("MM got timeout");
continue;
}
auto &&request_envelope = std::move(request_result.GetValue());
spdlog::info("MM got message to {}", request_envelope.to_address.ToString());
// If message is for the coordinator, cast it to subset and pass it to the coordinator
bool to_coordinator = coordinator_.GetAddress() == request_envelope.to_address;
spdlog::info("coordinator: {}", coordinator_.GetAddress().ToString());
if (to_coordinator) {
std::optional<CoordinatorMessages> conversion_attempt =
ConvertVariant<AllMessages, ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>,
AppendResponse, WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse>(
std::move(request_envelope.message));
MG_ASSERT(conversion_attempt.has_value(), "coordinator message conversion failed");
spdlog::info("got coordinator message");
CoordinatorMessages &&cm = std::move(conversion_attempt.value());
coordinator_.Handle(std::forward<CoordinatorMessages>(cm), request_envelope.request_id,
request_envelope.from_address);
continue;
}
bool to_sm = shard_manager_.GetAddress() == request_envelope.to_address;
spdlog::info("smm: {}", shard_manager_.GetAddress().ToString());
if (to_sm) {
std::optional<ShardManagerMessages> conversion_attempt =
ConvertVariant<AllMessages, WriteResponse<CoordinatorWriteResponses>>(std::move(request_envelope.message));
MG_ASSERT(conversion_attempt.has_value(), "shard manager message conversion failed");
spdlog::info("got shard manager message");
ShardManagerMessages &&smm = std::move(conversion_attempt.value());
shard_manager_.Receive(std::forward<ShardManagerMessages>(smm), request_envelope.request_id,
request_envelope.from_address);
continue;
}
// treat this as a message to a specific shard rsm and cast it accordingly
std::optional<ShardMessages> conversion_attempt =
ConvertVariant<AllMessages, ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>,
AppendResponse, WriteRequest<StorageWriteRequest>, VoteRequest, VoteResponse>(
std::move(request_envelope.message));
MG_ASSERT(conversion_attempt.has_value(), "shard rsm message conversion failed for {}",
request_envelope.to_address.ToString());
spdlog::info("got shard rsm message");
ShardMessages &&sm = std::move(conversion_attempt.value());
shard_manager_.Route(std::forward<ShardMessages>(sm), request_envelope.request_id, request_envelope.to_address,
request_envelope.from_address);
}
}
private:
Time Cron() {
spdlog::info("running MachineManager::Cron, address {}", io_.GetAddress().ToString());
return shard_manager_.Cron();
}
};
} // namespace memgraph::machine_manager

41
src/parser/CMakeLists.txt Normal file
View File

@ -0,0 +1,41 @@
## Generate Antlr openCypher parser
set(opencypher_frontend ${CMAKE_CURRENT_SOURCE_DIR}/opencypher)
set(opencypher_generated ${opencypher_frontend}/generated)
set(opencypher_lexer_grammar ${opencypher_frontend}/grammar/MemgraphCypherLexer.g4)
set(opencypher_parser_grammar ${opencypher_frontend}/grammar/MemgraphCypher.g4)
set(antlr_opencypher_generated_src
${opencypher_generated}/MemgraphCypherLexer.cpp
${opencypher_generated}/MemgraphCypher.cpp
${opencypher_generated}/MemgraphCypherBaseVisitor.cpp
${opencypher_generated}/MemgraphCypherVisitor.cpp
)
set(antlr_opencypher_generated_include
${opencypher_generated}/MemgraphCypherLexer.h
${opencypher_generated}/MemgraphCypher.h
${opencypher_generated}/MemgraphCypherBaseVisitor.h
${opencypher_generated}/MemgraphCypherVisitor.h
)
add_custom_command(
OUTPUT ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include}
COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
COMMAND
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-Dlanguage=Cpp -visitor -package antlropencypher
-o ${opencypher_generated}
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
DEPENDS
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
${opencypher_frontend}/grammar/CypherLexer.g4
${opencypher_frontend}/grammar/Cypher.g4)
add_custom_target(generated_opencypher_parser
DEPENDS ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include})
add_library(mg-parser STATIC ${antlr_opencypher_generated_src})
add_dependencies(mg-parser generated_opencypher_parser)
target_include_directories(mg-parser PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(mg-parser antlr4)

View File

@ -0,0 +1,391 @@
/*
* Copyright (c) 2015-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
parser grammar Cypher;
options { tokenVocab=CypherLexer; }
cypher : statement ';'? EOF ;
statement : query ;
query : cypherQuery
| indexQuery
| explainQuery
| profileQuery
| infoQuery
| constraintQuery
;
constraintQuery : ( CREATE | DROP ) CONSTRAINT ON constraint ;
constraint : '(' nodeName=variable ':' labelName ')' ASSERT EXISTS '(' constraintPropertyList ')'
| '(' nodeName=variable ':' labelName ')' ASSERT constraintPropertyList IS UNIQUE
| '(' nodeName=variable ':' labelName ')' ASSERT '(' constraintPropertyList ')' IS NODE KEY
;
constraintPropertyList : variable propertyLookup ( ',' variable propertyLookup )* ;
storageInfo : STORAGE INFO ;
indexInfo : INDEX INFO ;
constraintInfo : CONSTRAINT INFO ;
infoQuery : SHOW ( storageInfo | indexInfo | constraintInfo ) ;
explainQuery : EXPLAIN cypherQuery ;
profileQuery : PROFILE cypherQuery ;
cypherQuery : singleQuery ( cypherUnion )* ( queryMemoryLimit )? ;
indexQuery : createIndex | dropIndex;
singleQuery : clause ( clause )* ;
cypherUnion : ( UNION ALL singleQuery )
| ( UNION singleQuery )
;
clause : cypherMatch
| unwind
| merge
| create
| set
| cypherDelete
| remove
| with
| cypherReturn
| callProcedure
;
cypherMatch : OPTIONAL? MATCH pattern where? ;
unwind : UNWIND expression AS variable ;
merge : MERGE patternPart ( mergeAction )* ;
mergeAction : ( ON MATCH set )
| ( ON CREATE set )
;
create : CREATE pattern ;
set : SET setItem ( ',' setItem )* ;
setItem : ( propertyExpression '=' expression )
| ( variable '=' expression )
| ( variable '+=' expression )
| ( variable nodeLabels )
;
cypherDelete : DETACH? DELETE expression ( ',' expression )* ;
remove : REMOVE removeItem ( ',' removeItem )* ;
removeItem : ( variable nodeLabels )
| propertyExpression
;
with : WITH ( DISTINCT )? returnBody ( where )? ;
cypherReturn : RETURN ( DISTINCT )? returnBody ;
callProcedure : CALL procedureName '(' ( expression ( ',' expression )* )? ')' ( procedureMemoryLimit )? ( yieldProcedureResults )? ;
procedureName : symbolicName ( '.' symbolicName )* ;
yieldProcedureResults : YIELD ( '*' | ( procedureResult ( ',' procedureResult )* ) ) ;
memoryLimit : MEMORY ( UNLIMITED | LIMIT literal ( MB | KB ) ) ;
queryMemoryLimit : QUERY memoryLimit ;
procedureMemoryLimit : PROCEDURE memoryLimit ;
procedureResult : ( variable AS variable ) | variable ;
returnBody : returnItems ( order )? ( skip )? ( limit )? ;
returnItems : ( '*' ( ',' returnItem )* )
| ( returnItem ( ',' returnItem )* )
;
returnItem : ( expression AS variable )
| expression
;
order : ORDER BY sortItem ( ',' sortItem )* ;
skip : L_SKIP expression ;
limit : LIMIT expression ;
sortItem : expression ( ASCENDING | ASC | DESCENDING | DESC )? ;
where : WHERE expression ;
pattern : patternPart ( ',' patternPart )* ;
patternPart : ( variable '=' anonymousPatternPart )
| anonymousPatternPart
;
anonymousPatternPart : patternElement ;
patternElement : ( nodePattern ( patternElementChain )* )
| ( '(' patternElement ')' )
;
nodePattern : '(' ( variable )? ( nodeLabels )? ( properties )? ')' ;
patternElementChain : relationshipPattern nodePattern ;
relationshipPattern : ( leftArrowHead dash ( relationshipDetail )? dash rightArrowHead )
| ( leftArrowHead dash ( relationshipDetail )? dash )
| ( dash ( relationshipDetail )? dash rightArrowHead )
| ( dash ( relationshipDetail )? dash )
;
leftArrowHead : '<' | LeftArrowHeadPart ;
rightArrowHead : '>' | RightArrowHeadPart ;
dash : '-' | DashPart ;
relationshipDetail : '[' ( name=variable )? ( relationshipTypes )? ( variableExpansion )? properties ']'
| '[' ( name=variable )? ( relationshipTypes )? ( variableExpansion )? relationshipLambda ( total_weight=variable )? (relationshipLambda )? ']'
| '[' ( name=variable )? ( relationshipTypes )? ( variableExpansion )? (properties )* ( relationshipLambda total_weight=variable )? (relationshipLambda )? ']';
relationshipLambda: '(' traversed_edge=variable ',' traversed_node=variable '|' expression ')';
variableExpansion : '*' (BFS | WSHORTEST)? ( expression )? ( '..' ( expression )? )? ;
properties : mapLiteral
| parameter
;
relationshipTypes : ':' relTypeName ( '|' ':'? relTypeName )* ;
nodeLabels : nodeLabel ( nodeLabel )* ;
nodeLabel : ':' labelName ;
labelName : symbolicName ;
relTypeName : symbolicName ;
expression : expression12 ;
expression12 : expression11 ( OR expression11 )* ;
expression11 : expression10 ( XOR expression10 )* ;
expression10 : expression9 ( AND expression9 )* ;
expression9 : ( NOT )* expression8 ;
expression8 : expression7 ( partialComparisonExpression )* ;
expression7 : expression6 ( ( '+' expression6 ) | ( '-' expression6 ) )* ;
expression6 : expression5 ( ( '*' expression5 ) | ( '/' expression5 ) | ( '%' expression5 ) )* ;
expression5 : expression4 ( '^' expression4 )* ;
expression4 : ( ( '+' | '-' ) )* expression3a ;
expression3a : expression3b ( stringAndNullOperators )* ;
stringAndNullOperators : ( ( ( ( '=~' ) | ( IN ) | ( STARTS WITH ) | ( ENDS WITH ) | ( CONTAINS ) ) expression3b) | ( IS CYPHERNULL ) | ( IS NOT CYPHERNULL ) ) ;
expression3b : expression2a ( listIndexingOrSlicing )* ;
listIndexingOrSlicing : ( '[' expression ']' )
| ( '[' lower_bound=expression? '..' upper_bound=expression? ']' )
;
expression2a : expression2b ( nodeLabels )? ;
expression2b : atom ( propertyLookup )* ;
atom : literal
| parameter
| caseExpression
| ( COUNT '(' '*' ')' )
| listComprehension
| patternComprehension
| ( FILTER '(' filterExpression ')' )
| ( EXTRACT '(' extractExpression ')' )
| ( REDUCE '(' reduceExpression ')' )
| ( COALESCE '(' expression ( ',' expression )* ')' )
| ( ALL '(' filterExpression ')' )
| ( ANY '(' filterExpression ')' )
| ( NONE '(' filterExpression ')' )
| ( SINGLE '(' filterExpression ')' )
| relationshipsPattern
| parenthesizedExpression
| functionInvocation
| variable
;
literal : numberLiteral
| StringLiteral
| booleanLiteral
| CYPHERNULL
| mapLiteral
| listLiteral
;
booleanLiteral : TRUE
| FALSE
;
listLiteral : '[' ( expression ( ',' expression )* )? ']' ;
partialComparisonExpression : ( '=' expression7 )
| ( '<>' expression7 )
| ( '!=' expression7 )
| ( '<' expression7 )
| ( '>' expression7 )
| ( '<=' expression7 )
| ( '>=' expression7 )
;
parenthesizedExpression : '(' expression ')' ;
relationshipsPattern : nodePattern ( patternElementChain )+ ;
filterExpression : idInColl ( where )? ;
reduceExpression : accumulator=variable '=' initial=expression ',' idInColl '|' expression ;
extractExpression : idInColl '|' expression ;
idInColl : variable IN expression ;
functionInvocation : functionName '(' ( DISTINCT )? ( expression ( ',' expression )* )? ')' ;
functionName : symbolicName ( '.' symbolicName )* ;
listComprehension : '[' filterExpression ( '|' expression )? ']' ;
patternComprehension : '[' ( variable '=' )? relationshipsPattern ( WHERE expression )? '|' expression ']' ;
propertyLookup : '.' ( propertyKeyName ) ;
caseExpression : ( ( CASE ( caseAlternatives )+ ) | ( CASE test=expression ( caseAlternatives )+ ) ) ( ELSE else_expression=expression )? END ;
caseAlternatives : WHEN when_expression=expression THEN then_expression=expression ;
variable : symbolicName ;
numberLiteral : doubleLiteral
| integerLiteral
;
mapLiteral : '{' ( propertyKeyName ':' expression ( ',' propertyKeyName ':' expression )* )? '}' ;
parameter : '$' ( symbolicName | DecimalLiteral ) ;
propertyExpression : atom ( propertyLookup )+ ;
propertyKeyName : symbolicName ;
integerLiteral : DecimalLiteral
| OctalLiteral
| HexadecimalLiteral
;
createIndex : CREATE INDEX ON ':' labelName ( '(' propertyKeyName ')' )? ;
dropIndex : DROP INDEX ON ':' labelName ( '(' propertyKeyName ')' )? ;
doubleLiteral : FloatingLiteral ;
cypherKeyword : ALL
| AND
| ANY
| AS
| ASC
| ASCENDING
| ASSERT
| BFS
| BY
| CALL
| CASE
| CONSTRAINT
| CONTAINS
| COUNT
| CREATE
| CYPHERNULL
| DELETE
| DESC
| DESCENDING
| DETACH
| DISTINCT
| ELSE
| END
| ENDS
| EXISTS
| EXPLAIN
| EXTRACT
| FALSE
| FILTER
| IN
| INDEX
| INFO
| IS
| KEY
| LIMIT
| L_SKIP
| MATCH
| MERGE
| NODE
| NONE
| NOT
| ON
| OPTIONAL
| OR
| ORDER
| PROCEDURE
| PROFILE
| QUERY
| REDUCE
| REMOVE
| RETURN
| SET
| SHOW
| SINGLE
| STARTS
| STORAGE
| THEN
| TRUE
| UNION
| UNIQUE
| UNWIND
| WHEN
| WHERE
| WITH
| WSHORTEST
| XOR
| YIELD
;
symbolicName : UnescapedSymbolicName
| EscapedSymbolicName
| cypherKeyword
;

View File

@ -0,0 +1,208 @@
/*
* When changing this grammar make sure to update constants in
* src/parser/stripped_lexer_constants.hpp (kKeywords, kSpecialTokens
* and bitsets) if needed.
*/
lexer grammar CypherLexer ;
import UnicodeCategories ;
/* Skip whitespace and comments. */
Skipped : ( Whitespace | Comment ) -> skip ;
fragment Whitespace : '\u0020'
| [\u0009-\u000D]
| [\u001C-\u001F]
| '\u1680' | '\u180E'
| [\u2000-\u200A]
| '\u2028' | '\u2029'
| '\u205F'
| '\u3000'
| '\u00A0'
| '\u202F'
;
fragment Comment : '/*' .*? '*/'
| '//' ~[\r\n]*
;
/* Special symbols. */
LPAREN : '(' ;
RPAREN : ')' ;
LBRACK : '[' ;
RBRACK : ']' ;
LBRACE : '{' ;
RBRACE : '}' ;
COMMA : ',' ;
DOT : '.' ;
DOTS : '..' ;
COLON : ':' ;
SEMICOLON : ';' ;
DOLLAR : '$' ;
PIPE : '|' ;
EQ : '=' ;
LT : '<' ;
GT : '>' ;
LTE : '<=' ;
GTE : '>=' ;
NEQ1 : '<>' ;
NEQ2 : '!=' ;
SIM : '=~' ;
PLUS : '+' ;
MINUS : '-' ;
ASTERISK : '*' ;
SLASH : '/' ;
PERCENT : '%' ;
CARET : '^' ;
PLUS_EQ : '+=' ;
/* Some random unicode characters that can be used to draw arrows. */
LeftArrowHeadPart : '⟨' | '〈' | '﹤' | '' ;
RightArrowHeadPart : '⟩' | '〉' | '﹥' | '' ;
DashPart : '­' | '' | '' | '' | '' | '—' | '―'
| '' | '' | '﹣' | ''
;
/* Cypher reserved words. */
ALL : A L L ;
AND : A N D ;
ANY : A N Y ;
AS : A S ;
ASC : A S C ;
ASCENDING : A S C E N D I N G ;
ASSERT : A S S E R T ;
BFS : B F S ;
BY : B Y ;
CALL : C A L L ;
CASE : C A S E ;
COALESCE : C O A L E S C E ;
CONSTRAINT : C O N S T R A I N T ;
CONTAINS : C O N T A I N S ;
COUNT : C O U N T ;
CREATE : C R E A T E ;
CYPHERNULL : N U L L ;
DELETE : D E L E T E ;
DESC : D E S C ;
DESCENDING : D E S C E N D I N G ;
DETACH : D E T A C H ;
DISTINCT : D I S T I N C T ;
DROP : D R O P ;
ELSE : E L S E ;
END : E N D ;
ENDS : E N D S ;
EXISTS : E X I S T S ;
EXPLAIN : E X P L A I N ;
EXTRACT : E X T R A C T ;
FALSE : F A L S E ;
FILTER : F I L T E R ;
IN : I N ;
INDEX : I N D E X ;
INFO : I N F O ;
IS : I S ;
KB : K B ;
KEY : K E Y ;
LIMIT : L I M I T ;
L_SKIP : S K I P ;
MATCH : M A T C H ;
MB : M B ;
MEMORY : M E M O R Y ;
MERGE : M E R G E ;
NODE : N O D E ;
NONE : N O N E ;
NOT : N O T ;
ON : O N ;
OPTIONAL : O P T I O N A L ;
OR : O R ;
ORDER : O R D E R ;
PROCEDURE : P R O C E D U R E ;
PROFILE : P R O F I L E ;
QUERY : Q U E R Y ;
REDUCE : R E D U C E ;
REMOVE : R E M O V E ;
RETURN : R E T U R N ;
SET : S E T ;
SHOW : S H O W ;
SINGLE : S I N G L E ;
STARTS : S T A R T S ;
STORAGE : S T O R A G E ;
THEN : T H E N ;
TRUE : T R U E ;
UNION : U N I O N ;
UNIQUE : U N I Q U E ;
UNLIMITED : U N L I M I T E D ;
UNWIND : U N W I N D ;
WHEN : W H E N ;
WHERE : W H E R E ;
WITH : W I T H ;
WSHORTEST : W S H O R T E S T ;
XOR : X O R ;
YIELD : Y I E L D ;
/* Double and single quoted string literals. */
StringLiteral : '"' ( ~[\\"] | EscapeSequence )* '"'
| '\'' ( ~[\\'] | EscapeSequence )* '\''
;
fragment EscapeSequence : '\\' ( B | F | N | R | T | '\\' | '\'' | '"' )
| '\\u' HexDigit HexDigit HexDigit HexDigit
| '\\U' HexDigit HexDigit HexDigit HexDigit
HexDigit HexDigit HexDigit HexDigit
;
/* Number literals. */
DecimalLiteral : '0' | NonZeroDigit ( DecDigit )* ;
OctalLiteral : '0' ( OctDigit )+ ;
HexadecimalLiteral : '0x' ( HexDigit )+ ;
FloatingLiteral : DecDigit* '.' DecDigit+ ( E '-'? DecDigit+ )?
| DecDigit+ ( '.' DecDigit* )? ( E '-'? DecDigit+ )
| DecDigit+ ( E '-'? DecDigit+ )
;
fragment NonZeroDigit : [1-9] ;
fragment DecDigit : [0-9] ;
fragment OctDigit : [0-7] ;
fragment HexDigit : [0-9] | [a-f] | [A-F] ;
/* Symbolic names. */
UnescapedSymbolicName : IdentifierStart ( IdentifierPart )* ;
EscapedSymbolicName : ( '`' ~[`]* '`' )+ ;
/**
* Based on the unicode identifier and pattern syntax
* (http://www.unicode.org/reports/tr31/)
* and extended with a few characters.
*/
IdentifierStart : ID_Start | Pc ;
IdentifierPart : ID_Continue | Sc ;
/* Hack for case-insensitive reserved words */
fragment A : 'A' | 'a' ;
fragment B : 'B' | 'b' ;
fragment C : 'C' | 'c' ;
fragment D : 'D' | 'd' ;
fragment E : 'E' | 'e' ;
fragment F : 'F' | 'f' ;
fragment G : 'G' | 'g' ;
fragment H : 'H' | 'h' ;
fragment I : 'I' | 'i' ;
fragment J : 'J' | 'j' ;
fragment K : 'K' | 'k' ;
fragment L : 'L' | 'l' ;
fragment M : 'M' | 'm' ;
fragment N : 'N' | 'n' ;
fragment O : 'O' | 'o' ;
fragment P : 'P' | 'p' ;
fragment Q : 'Q' | 'q' ;
fragment R : 'R' | 'r' ;
fragment S : 'S' | 's' ;
fragment T : 'T' | 't' ;
fragment U : 'U' | 'u' ;
fragment V : 'V' | 'v' ;
fragment W : 'W' | 'w' ;
fragment X : 'X' | 'x' ;
fragment Y : 'Y' | 'y' ;
fragment Z : 'Z' | 'z' ;

View File

@ -0,0 +1,399 @@
/*
* Copyright 2021 Memgraph Ltd.
*
* Use of this software is governed by the Business Source License
* included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
* License, and you may not use this file except in compliance with the Business Source License.
*
* As of the Change Date specified in that file, in accordance with
* the Business Source License, use of this software will be governed
* by the Apache License, Version 2.0, included in the file
* licenses/APL.txt.
*/
/* Memgraph specific part of Cypher grammar with enterprise features. */
parser grammar MemgraphCypher ;
options { tokenVocab=MemgraphCypherLexer; }
import Cypher ;
memgraphCypherKeyword : cypherKeyword
| AFTER
| ALTER
| ASYNC
| AUTH
| BAD
| BATCH_INTERVAL
| BATCH_LIMIT
| BATCH_SIZE
| BEFORE
| BOOTSTRAP_SERVERS
| CHECK
| CLEAR
| COMMIT
| COMMITTED
| CONFIG
| CONFIGS
| CONSUMER_GROUP
| CREDENTIALS
| CSV
| DATA
| DELIMITER
| DATABASE
| DENY
| DROP
| DUMP
| EXECUTE
| FREE
| FROM
| FOR
| FOREACH
| GLOBAL
| GRANT
| HEADER
| IDENTIFIED
| ISOLATION
| KAFKA
| LEVEL
| LOAD
| LOCK
| MAIN
| MODE
| NEXT
| NO
| PASSWORD
| PULSAR
| PORT
| PRIVILEGES
| READ
| REGISTER
| REPLICA
| REPLICAS
| REPLICATION
| REVOKE
| ROLE
| ROLES
| QUOTE
| SCHEMA
| SCHEMAS
| SESSION
| SETTING
| SETTINGS
| SNAPSHOT
| START
| STATS
| STREAM
| STREAMS
| SYNC
| TIMEOUT
| TO
| TOPICS
| TRANSACTION
| TRANSFORM
| TRIGGER
| TRIGGERS
| UNCOMMITTED
| UNLOCK
| UPDATE
| USER
| USERS
| VERSION
;
symbolicName : UnescapedSymbolicName
| EscapedSymbolicName
| memgraphCypherKeyword
;
query : cypherQuery
| indexQuery
| explainQuery
| profileQuery
| infoQuery
| constraintQuery
| authQuery
| dumpQuery
| replicationQuery
| lockPathQuery
| freeMemoryQuery
| triggerQuery
| isolationLevelQuery
| createSnapshotQuery
| streamQuery
| settingQuery
| versionQuery
| schemaQuery
;
authQuery : createRole
| dropRole
| showRoles
| createUser
| setPassword
| dropUser
| showUsers
| setRole
| clearRole
| grantPrivilege
| denyPrivilege
| revokePrivilege
| showPrivileges
| showRoleForUser
| showUsersForRole
;
replicationQuery : setReplicationRole
| showReplicationRole
| registerReplica
| dropReplica
| showReplicas
;
triggerQuery : createTrigger
| dropTrigger
| showTriggers
;
clause : cypherMatch
| unwind
| merge
| create
| set
| cypherDelete
| remove
| with
| cypherReturn
| callProcedure
| loadCsv
| foreach
;
updateClause : set
| remove
| create
| merge
| cypherDelete
| foreach
;
foreach : FOREACH '(' variable IN expression '|' updateClause+ ')' ;
streamQuery : checkStream
| createStream
| dropStream
| startStream
| startAllStreams
| stopStream
| stopAllStreams
| showStreams
;
settingQuery : setSetting
| showSetting
| showSettings
;
schemaQuery : showSchema
| showSchemas
| createSchema
| dropSchema
;
loadCsv : LOAD CSV FROM csvFile ( WITH | NO ) HEADER
( IGNORE BAD ) ?
( DELIMITER delimiter ) ?
( QUOTE quote ) ?
AS rowVar ;
csvFile : literal ;
delimiter : literal ;
quote : literal ;
rowVar : variable ;
userOrRoleName : symbolicName ;
createRole : CREATE ROLE role=userOrRoleName ;
dropRole : DROP ROLE role=userOrRoleName ;
showRoles : SHOW ROLES ;
createUser : CREATE USER user=userOrRoleName
( IDENTIFIED BY password=literal )? ;
setPassword : SET PASSWORD FOR user=userOrRoleName TO password=literal;
dropUser : DROP USER user=userOrRoleName ;
showUsers : SHOW USERS ;
setRole : SET ROLE FOR user=userOrRoleName TO role=userOrRoleName;
clearRole : CLEAR ROLE FOR user=userOrRoleName ;
grantPrivilege : GRANT ( ALL PRIVILEGES | privileges=privilegeList ) TO userOrRole=userOrRoleName ;
denyPrivilege : DENY ( ALL PRIVILEGES | privileges=privilegeList ) TO userOrRole=userOrRoleName ;
revokePrivilege : REVOKE ( ALL PRIVILEGES | privileges=privilegeList ) FROM userOrRole=userOrRoleName ;
privilege : CREATE
| DELETE
| MATCH
| MERGE
| SET
| REMOVE
| INDEX
| STATS
| AUTH
| CONSTRAINT
| DUMP
| REPLICATION
| READ_FILE
| FREE_MEMORY
| TRIGGER
| CONFIG
| DURABILITY
| STREAM
| MODULE_READ
| MODULE_WRITE
| WEBSOCKET
| SCHEMA
;
privilegeList : privilege ( ',' privilege )* ;
showPrivileges : SHOW PRIVILEGES FOR userOrRole=userOrRoleName ;
showRoleForUser : SHOW ROLE FOR user=userOrRoleName ;
showUsersForRole : SHOW USERS FOR role=userOrRoleName ;
dumpQuery: DUMP DATABASE ;
setReplicationRole : SET REPLICATION ROLE TO ( MAIN | REPLICA )
( WITH PORT port=literal ) ? ;
showReplicationRole : SHOW REPLICATION ROLE ;
replicaName : symbolicName ;
socketAddress : literal ;
registerReplica : REGISTER REPLICA replicaName ( SYNC | ASYNC )
TO socketAddress ;
dropReplica : DROP REPLICA replicaName ;
showReplicas : SHOW REPLICAS ;
lockPathQuery : ( LOCK | UNLOCK ) DATA DIRECTORY ;
freeMemoryQuery : FREE MEMORY ;
triggerName : symbolicName ;
triggerStatement : .*? ;
emptyVertex : '(' ')' ;
emptyEdge : dash dash rightArrowHead ;
createTrigger : CREATE TRIGGER triggerName ( ON ( emptyVertex | emptyEdge ) ? ( CREATE | UPDATE | DELETE ) ) ?
( AFTER | BEFORE ) COMMIT EXECUTE triggerStatement ;
dropTrigger : DROP TRIGGER triggerName ;
showTriggers : SHOW TRIGGERS ;
isolationLevel : SNAPSHOT ISOLATION | READ COMMITTED | READ UNCOMMITTED ;
isolationLevelScope : GLOBAL | SESSION | NEXT ;
isolationLevelQuery : SET isolationLevelScope TRANSACTION ISOLATION LEVEL isolationLevel ;
createSnapshotQuery : CREATE SNAPSHOT ;
streamName : symbolicName ;
symbolicNameWithMinus : symbolicName ( MINUS symbolicName )* ;
symbolicNameWithDotsAndMinus: symbolicNameWithMinus ( DOT symbolicNameWithMinus )* ;
symbolicTopicNames : symbolicNameWithDotsAndMinus ( COMMA symbolicNameWithDotsAndMinus )* ;
topicNames : symbolicTopicNames | literal ;
commonCreateStreamConfig : TRANSFORM transformationName=procedureName
| BATCH_INTERVAL batchInterval=literal
| BATCH_SIZE batchSize=literal
;
createStream : kafkaCreateStream | pulsarCreateStream ;
configKeyValuePair : literal ':' literal ;
configMap : '{' ( configKeyValuePair ( ',' configKeyValuePair )* )? '}' ;
kafkaCreateStreamConfig : TOPICS topicNames
| CONSUMER_GROUP consumerGroup=symbolicNameWithDotsAndMinus
| BOOTSTRAP_SERVERS bootstrapServers=literal
| CONFIGS configsMap=configMap
| CREDENTIALS credentialsMap=configMap
| commonCreateStreamConfig
;
kafkaCreateStream : CREATE KAFKA STREAM streamName ( kafkaCreateStreamConfig ) * ;
pulsarCreateStreamConfig : TOPICS topicNames
| SERVICE_URL serviceUrl=literal
| commonCreateStreamConfig
;
pulsarCreateStream : CREATE PULSAR STREAM streamName ( pulsarCreateStreamConfig ) * ;
dropStream : DROP STREAM streamName ;
startStream : START STREAM streamName ( BATCH_LIMIT batchLimit=literal ) ? ( TIMEOUT timeout=literal ) ? ;
startAllStreams : START ALL STREAMS ;
stopStream : STOP STREAM streamName ;
stopAllStreams : STOP ALL STREAMS ;
showStreams : SHOW STREAMS ;
checkStream : CHECK STREAM streamName ( BATCH_LIMIT batchLimit=literal ) ? ( TIMEOUT timeout=literal ) ? ;
settingName : literal ;
settingValue : literal ;
setSetting : SET DATABASE SETTING settingName TO settingValue ;
showSetting : SHOW DATABASE SETTING settingName ;
showSettings : SHOW DATABASE SETTINGS ;
versionQuery : SHOW VERSION ;
showSchema : SHOW SCHEMA ON ':' labelName ;
showSchemas : SHOW SCHEMAS ;
propertyType : symbolicName ;
propertyKeyTypePair : propertyKeyName propertyType ;
schemaPropertyMap : '(' propertyKeyTypePair ( ',' propertyKeyTypePair )* ')' ;
createSchema : CREATE SCHEMA ON ':' labelName schemaPropertyMap ;
dropSchema : DROP SCHEMA ON ':' labelName ;

View File

@ -0,0 +1,118 @@
/*
* Copyright 2021 Memgraph Ltd.
*
* Use of this software is governed by the Business Source License
* included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
* License, and you may not use this file except in compliance with the Business Source License.
*
* As of the Change Date specified in that file, in accordance with
* the Business Source License, use of this software will be governed
* by the Apache License, Version 2.0, included in the file
* licenses/APL.txt.
*/
/* Memgraph specific Cypher reserved words used for enterprise features. */
/*
* When changing this grammar make sure to update constants in
* src/parser/stripped_lexer_constants.hpp (kKeywords, kSpecialTokens
* and bitsets) if needed.
*/
lexer grammar MemgraphCypherLexer ;
import CypherLexer ;
UNDERSCORE : '_' ;
AFTER : A F T E R ;
ALTER : A L T E R ;
ASYNC : A S Y N C ;
AUTH : A U T H ;
BAD : B A D ;
BATCH_INTERVAL : B A T C H UNDERSCORE I N T E R V A L ;
BATCH_LIMIT : B A T C H UNDERSCORE L I M I T ;
BATCH_SIZE : B A T C H UNDERSCORE S I Z E ;
BEFORE : B E F O R E ;
BOOTSTRAP_SERVERS : B O O T S T R A P UNDERSCORE S E R V E R S ;
CHECK : C H E C K ;
CLEAR : C L E A R ;
COMMIT : C O M M I T ;
COMMITTED : C O M M I T T E D ;
CONFIG : C O N F I G ;
CONFIGS : C O N F I G S;
CONSUMER_GROUP : C O N S U M E R UNDERSCORE G R O U P ;
CREDENTIALS : C R E D E N T I A L S ;
CSV : C S V ;
DATA : D A T A ;
DELIMITER : D E L I M I T E R ;
DATABASE : D A T A B A S E ;
DENY : D E N Y ;
DIRECTORY : D I R E C T O R Y ;
DROP : D R O P ;
DUMP : D U M P ;
DURABILITY : D U R A B I L I T Y ;
EXECUTE : E X E C U T E ;
FOR : F O R ;
FOREACH : F O R E A C H;
FREE : F R E E ;
FREE_MEMORY : F R E E UNDERSCORE M E M O R Y ;
FROM : F R O M ;
GLOBAL : G L O B A L ;
GRANT : G R A N T ;
GRANTS : G R A N T S ;
HEADER : H E A D E R ;
IDENTIFIED : I D E N T I F I E D ;
IGNORE : I G N O R E ;
ISOLATION : I S O L A T I O N ;
KAFKA : K A F K A ;
LEVEL : L E V E L ;
LOAD : L O A D ;
LOCK : L O C K ;
MAIN : M A I N ;
MODE : M O D E ;
MODULE_READ : M O D U L E UNDERSCORE R E A D ;
MODULE_WRITE : M O D U L E UNDERSCORE W R I T E ;
NEXT : N E X T ;
NO : N O ;
PASSWORD : P A S S W O R D ;
PORT : P O R T ;
PRIVILEGES : P R I V I L E G E S ;
PULSAR : P U L S A R ;
READ : R E A D ;
READ_FILE : R E A D UNDERSCORE F I L E ;
REGISTER : R E G I S T E R ;
REPLICA : R E P L I C A ;
REPLICAS : R E P L I C A S ;
REPLICATION : R E P L I C A T I O N ;
REVOKE : R E V O K E ;
ROLE : R O L E ;
ROLES : R O L E S ;
QUOTE : Q U O T E ;
SCHEMA : S C H E M A ;
SCHEMAS : S C H E M A S ;
SERVICE_URL : S E R V I C E UNDERSCORE U R L ;
SESSION : S E S S I O N ;
SETTING : S E T T I N G ;
SETTINGS : S E T T I N G S ;
SNAPSHOT : S N A P S H O T ;
START : S T A R T ;
STATS : S T A T S ;
STOP : S T O P ;
STREAM : S T R E A M ;
STREAMS : S T R E A M S ;
SYNC : S Y N C ;
TIMEOUT : T I M E O U T ;
TO : T O ;
TOPICS : T O P I C S;
TRANSACTION : T R A N S A C T I O N ;
TRANSFORM : T R A N S F O R M ;
TRIGGER : T R I G G E R ;
TRIGGERS : T R I G G E R S ;
UNCOMMITTED : U N C O M M I T T E D ;
UNLOCK : U N L O C K ;
UPDATE : U P D A T E ;
USER : U S E R ;
USERS : U S E R S ;
VERSION : V E R S I O N ;
WEBSOCKET : W E B S O C K E T ;

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,85 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <string>
#include "antlr4-runtime.h"
#include "utils/exceptions.hpp"
#include "parser/opencypher/generated/MemgraphCypher.h"
#include "parser/opencypher/generated/MemgraphCypherLexer.h"
#include "utils/concepts.hpp"
namespace memgraph::frontend::opencypher {
class SyntaxException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SyntaxException() : SyntaxException("") {}
};
/**
* Generates openCypher AST
* This thing must me a class since parser.cypher() returns pointer and there is
* no way for us to get ownership over the object.
*/
enum class ParserOpTag : uint8_t {
CYPHER, EXPRESSION
};
template<ParserOpTag Tag = ParserOpTag::CYPHER>
class Parser {
public:
/**
* @param query incoming query that has to be compiled into query plan
* the first step is to generate AST
*/
Parser(const std::string query) : query_(std::move(query)) {
parser_.removeErrorListeners();
parser_.addErrorListener(&error_listener_);
if constexpr(Tag == ParserOpTag::CYPHER) {
tree_ = parser_.cypher();
}
else {
tree_ = parser_.expression();
}
if (parser_.getNumberOfSyntaxErrors()) {
throw SyntaxException(error_listener_.error_);
}
}
auto tree() { return tree_; }
private:
class FirstMessageErrorListener : public antlr4::BaseErrorListener {
void syntaxError(antlr4::Recognizer *, antlr4::Token *, size_t line, size_t position, const std::string &message,
std::exception_ptr) override {
if (error_.empty()) {
error_ = "line " + std::to_string(line) + ":" + std::to_string(position + 1) + " " + message;
}
}
public:
std::string error_;
};
FirstMessageErrorListener error_listener_;
std::string query_;
antlr4::ANTLRInputStream input_{query_};
antlropencypher::MemgraphCypherLexer lexer_{&input_};
antlr4::CommonTokenStream tokens_{&lexer_};
// generate ast
antlropencypher::MemgraphCypher parser_{&tokens_};
antlr4::tree::ParseTree *tree_ = nullptr;
};
} // namespace memgraph::frontend::opencypher

File diff suppressed because it is too large Load Diff

View File

@ -51,7 +51,6 @@ class EdgeAccessor final {
public:
storage::EdgeAccessor impl_;
public:
explicit EdgeAccessor(storage::EdgeAccessor impl) : impl_(std::move(impl)) {}
bool IsVisible(storage::View view) const { return impl_.IsVisible(view); }
@ -97,7 +96,6 @@ class VertexAccessor final {
static EdgeAccessor MakeEdgeAccessor(const storage::EdgeAccessor impl) { return EdgeAccessor(impl); }
public:
explicit VertexAccessor(storage::VertexAccessor impl) : impl_(impl) {}
bool IsVisible(storage::View view) const { return impl_.IsVisible(view); }

View File

@ -204,7 +204,7 @@ const trie::Trie kKeywords = {"union",
"pulsar",
"service_url",
"version",
"websocket"
"websocket",
"foreach"};
// Unicode codepoints that are allowed at the start of the unescaped name.

View File

@ -114,4 +114,4 @@ std::string ExecutionStatsKeyToString(const ExecutionStats::Key key) {
}
}
} // namespace memgraph::query
} // namespace memgraph::query

View File

@ -368,13 +368,12 @@ VertexAccessor &CreateExpand::CreateExpandCursor::OtherVertex(Frame &frame, Exec
TypedValue &dest_node_value = frame[self_.node_info_.symbol];
ExpectType(self_.node_info_.symbol, dest_node_value, TypedValue::Type::Vertex);
return dest_node_value.ValueVertex();
} else {
auto &created_vertex = CreateLocalVertex(self_.node_info_, &frame, context);
if (context.trigger_context_collector) {
context.trigger_context_collector->RegisterCreatedObject(created_vertex);
}
return created_vertex;
}
auto &created_vertex = CreateLocalVertex(self_.node_info_, &frame, context);
if (context.trigger_context_collector) {
context.trigger_context_collector->RegisterCreatedObject(created_vertex);
}
return created_vertex;
}
template <class TVerticesFun>

View File

@ -0,0 +1,59 @@
define_add_lcp(add_lcp_query lcp_query_v2_cpp_files generated_lcp_query_v2_files)
add_lcp_query(frontend/ast/ast.lcp)
add_lcp_query(plan/operator.lcp)
add_custom_target(generate_lcp_query_v2 DEPENDS ${generated_lcp_query_v2_files})
set(mg_query_v2_sources
${lcp_query_v2_cpp_files}
common.cpp
cypher_query_interpreter.cpp
dump.cpp
frontend/semantic/required_privileges.cpp
frontend/stripped.cpp
interpret/awesome_memgraph_functions.cpp
interpreter.cpp
metadata.cpp
plan/operator.cpp
plan/preprocess.cpp
plan/pretty_print.cpp
plan/profile.cpp
plan/read_write_type_checker.cpp
plan/rewrite/index_lookup.cpp
plan/rule_based_planner.cpp
plan/variable_start_planner.cpp
# procedure/mg_procedure_impl.cpp
# procedure/mg_procedure_helpers.cpp
# procedure/module.cpp
# procedure/py_module.cpp
serialization/property_value.cpp
# stream/streams.cpp
# stream/sources.cpp
# stream/common.cpp
# trigger.cpp
# trigger_context.cpp
bindings/typed_value.cpp
accessors.cpp)
find_package(Boost REQUIRED)
add_library(mg-query-v2 STATIC ${mg_query_v2_sources})
add_dependencies(mg-query-v2 generate_lcp_query_v2)
target_include_directories(mg-query-v2 PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_include_directories(mg-query-v2 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bindings)
target_link_libraries(mg-query-v2 dl cppitertools Boost::headers)
target_link_libraries(mg-query-v2 mg-integrations-pulsar mg-integrations-kafka mg-storage-v3 mg-license mg-utils mg-kvstore mg-memory)
target_link_libraries(mg-query-v2 mg-expr)
if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")
endif()
if("${MG_PYTHON_VERSION}" STREQUAL "")
find_package(Python3 3.5 REQUIRED COMPONENTS Development)
else()
find_package(Python3 "${MG_PYTHON_VERSION}" EXACT REQUIRED COMPONENTS Development)
endif()
target_link_libraries(mg-query-v2 Python3::Python)

View File

@ -0,0 +1,75 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/v2/accessors.hpp"
#include "query/v2/requests.hpp"
namespace memgraph::query::v2::accessors {
EdgeAccessor::EdgeAccessor(Edge edge, std::vector<std::pair<PropertyId, Value>> props)
: edge(std::move(edge)), properties(std::move(props)) {}
uint64_t EdgeAccessor::EdgeType() const { return edge.type.id; }
std::vector<std::pair<PropertyId, Value>> EdgeAccessor::Properties() const {
return properties;
// std::map<std::string, TypedValue> res;
// for (const auto &[name, value] : *properties) {
// res[name] = ValueToTypedValue(value);
// }
// return res;
}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
Value EdgeAccessor::GetProperty(const std::string & /*prop_name*/) const {
// TODO(kostasrim) fix this
return {};
}
Edge EdgeAccessor::GetEdge() const { return edge; }
VertexAccessor EdgeAccessor::To() const { return VertexAccessor(Vertex{edge.dst}, {}); }
VertexAccessor EdgeAccessor::From() const { return VertexAccessor(Vertex{edge.src}, {}); }
VertexAccessor::VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props)
: vertex(std::move(v)), properties(std::move(props)) {}
std::vector<Label> VertexAccessor::Labels() const { return vertex.labels; }
bool VertexAccessor::HasLabel(Label &label) const {
return std::find_if(vertex.labels.begin(), vertex.labels.end(),
[label](const auto &l) { return l.id == label.id; }) != vertex.labels.end();
}
std::vector<std::pair<PropertyId, Value>> VertexAccessor::Properties() const {
// std::map<std::string, TypedValue> res;
// for (const auto &[name, value] : *properties) {
// res[name] = ValueToTypedValue(value);
// }
// return res;
return properties;
}
Value VertexAccessor::GetProperty(PropertyId prop_id) const {
return std::find_if(properties.begin(), properties.end(), [&](auto &pr) { return prop_id == pr.first; })->second;
// return ValueToTypedValue(properties[prop_name]);
}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
Value VertexAccessor::GetProperty(const std::string & /*prop_name*/) const {
// TODO(kostasrim) Add string mapping
return {};
// return ValueToTypedValue(properties[prop_name]);
}
msgs::Vertex VertexAccessor::GetVertex() const { return vertex; }
} // namespace memgraph::query::v2::accessors

187
src/query/v2/accessors.hpp Normal file
View File

@ -0,0 +1,187 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <optional>
#include <utility>
#include <vector>
#include "query/exceptions.hpp"
#include "query/v2/requests.hpp"
#include "storage/v3/view.hpp"
#include "utils/bound.hpp"
#include "utils/exceptions.hpp"
#include "utils/memory.hpp"
#include "utils/memory_tracker.hpp"
namespace memgraph::query::v2::accessors {
using Value = memgraph::msgs::Value;
using Edge = memgraph::msgs::Edge;
using Vertex = memgraph::msgs::Vertex;
using Label = memgraph::msgs::Label;
using PropertyId = memgraph::msgs::PropertyId;
class VertexAccessor;
class EdgeAccessor final {
public:
EdgeAccessor(Edge edge, std::vector<std::pair<PropertyId, Value>> props);
uint64_t EdgeType() const;
std::vector<std::pair<PropertyId, Value>> Properties() const;
Value GetProperty(const std::string &prop_name) const;
Edge GetEdge() const;
// Dummy function
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
inline size_t CypherId() const { return 10; }
// bool HasSrcAccessor const { return src == nullptr; }
// bool HasDstAccessor const { return dst == nullptr; }
VertexAccessor To() const;
VertexAccessor From() const;
friend bool operator==(const EdgeAccessor &lhs, const EdgeAccessor &rhs) {
return lhs.edge == rhs.edge && lhs.properties == rhs.properties;
}
friend bool operator!=(const EdgeAccessor &lhs, const EdgeAccessor &rhs) { return !(lhs == rhs); }
private:
Edge edge;
std::vector<std::pair<PropertyId, Value>> properties;
};
class VertexAccessor final {
public:
using PropertyId = msgs::PropertyId;
using Label = msgs::Label;
VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props);
std::vector<Label> Labels() const;
bool HasLabel(Label &label) const;
std::vector<std::pair<PropertyId, Value>> Properties() const;
Value GetProperty(PropertyId prop_id) const;
Value GetProperty(const std::string &prop_name) const;
msgs::Vertex GetVertex() const;
// Dummy function
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
inline size_t CypherId() const { return 10; }
// auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
// auto maybe_edges = impl_.InEdges(view, edge_types);
// if (maybe_edges.HasError()) return maybe_edges.GetError();
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
// }
//
// auto InEdges(storage::View view) const { return InEdges(view, {}); }
//
// auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types, const VertexAccessor &dest)
// const
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
// auto maybe_edges = impl_.InEdges(view, edge_types, &dest.impl_);
// if (maybe_edges.HasError()) return maybe_edges.GetError();
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
// }
//
// auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
// auto maybe_edges = impl_.OutEdges(view, edge_types);
// if (maybe_edges.HasError()) return maybe_edges.GetError();
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
// }
//
// auto OutEdges(storage::View view) const { return OutEdges(view, {}); }
//
// auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types,
// const VertexAccessor &dest) const
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
// auto maybe_edges = impl_.OutEdges(view, edge_types, &dest.impl_);
// if (maybe_edges.HasError()) return maybe_edges.GetError();
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
// }
// storage::Result<size_t> InDegree(storage::View view) const { return impl_.InDegree(view); }
//
// storage::Result<size_t> OutDegree(storage::View view) const { return impl_.OutDegree(view); }
//
friend bool operator==(const VertexAccessor &lhs, const VertexAccessor &rhs) {
return lhs.vertex == rhs.vertex && lhs.properties == rhs.properties;
}
friend bool operator!=(const VertexAccessor &lhs, const VertexAccessor &rhs) { return !(lhs == rhs); }
private:
Vertex vertex;
std::vector<std::pair<PropertyId, Value>> properties;
};
// inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_.ToVertex()); }
// inline VertexAccessor EdgeAccessor::From() const { return VertexAccessor(impl_.FromVertex()); }
// Highly mocked interface. Won't work if used.
class Path {
public:
// Empty for now
explicit Path(const VertexAccessor & /*vertex*/, utils::MemoryResource *memory = utils::NewDeleteResource())
: mem(memory) {}
template <typename... TOthers>
explicit Path(const VertexAccessor &vertex, const TOthers &...others) {}
template <typename... TOthers>
Path(std::allocator_arg_t /*unused*/, utils::MemoryResource *memory, const VertexAccessor &vertex,
const TOthers &...others) {}
Path(const Path & /*other*/) {}
Path(const Path & /*other*/, utils::MemoryResource *memory) : mem(memory) {}
Path(Path && /*other*/) noexcept {}
Path(Path && /*other*/, utils::MemoryResource *memory) : mem(memory) {}
Path &operator=(const Path &path) {
if (this == &path) {
return *this;
}
return *this;
}
Path &operator=(Path &&path) noexcept {
if (this == &path) {
return *this;
}
return *this;
}
~Path() {}
friend bool operator==(const Path & /*lhs*/, const Path & /*rhs*/) { return true; };
utils::MemoryResource *GetMemoryResource() { return mem; }
private:
utils::MemoryResource *mem = utils::NewDeleteResource();
};
} // namespace memgraph::query::v2::accessors

View File

@ -0,0 +1,29 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/frontend/ast/ast.hpp"
namespace memgraph::query::v2 {
class AuthChecker {
public:
virtual bool IsUserAuthorized(const std::optional<std::string> &username,
const std::vector<query::v2::AuthQuery::Privilege> &privileges) const = 0;
};
class AllowEverythingAuthChecker final : public query::v2::AuthChecker {
bool IsUserAuthorized(const std::optional<std::string> & /*username*/,
const std::vector<query::v2::AuthQuery::Privilege> & /*privileges*/) const override {
return true;
}
};
} // namespace memgraph::query::v2

View File

@ -0,0 +1,16 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/bindings/bindings.hpp"
#include "expr/ast/ast_visitor.hpp"

View File

@ -0,0 +1,15 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#define MG_AST_INCLUDE_PATH "query/v2/frontend/ast/ast.hpp" // NOLINT(cppcoreguidelines-macro-usage)
#define MG_INJECTED_NAMESPACE_NAME memgraph::query::v2 // NOLINT(cppcoreguidelines-macro-usage)

View File

@ -0,0 +1,20 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/bindings/bindings.hpp"
#include "expr/ast/cypher_main_visitor.hpp"
namespace memgraph::query::v2 {
using CypherMainVisitor = memgraph::expr::CypherMainVisitor;
} // namespace memgraph::query::v2

View File

@ -0,0 +1,44 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/bindings/bindings.hpp"
#include "expr/interpret/eval.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/context.hpp"
#include "query/v2/conversions.hpp"
#include "query/v2/db_accessor.hpp"
#include "query/v2/requests.hpp"
#include "storage/v3/conversions.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/view.hpp"
namespace memgraph::query::v2 {
inline const auto lam = [](const auto &val) { return ValueToTypedValue(val); };
namespace detail {
class Callable {
public:
auto operator()(const memgraph::storage::v3::PropertyValue &val) const {
return memgraph::storage::v3::PropertyToTypedValue<TypedValue>(val);
};
auto operator()(const msgs::Value &val) const { return ValueToTypedValue(val); };
};
} // namespace detail
using ExpressionEvaluator =
memgraph::expr::ExpressionEvaluator<TypedValue, memgraph::query::v2::EvaluationContext, DbAccessor,
storage::v3::View, storage::v3::LabelId, msgs::Value, detail::Callable,
memgraph::storage::v3::Error, memgraph::expr::QueryEngineTag>;
} // namespace memgraph::query::v2

View File

@ -0,0 +1,21 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/bindings/bindings.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "expr/interpret/frame.hpp"
namespace memgraph::query::v2 {
using Frame = memgraph::expr::Frame<TypedValue>;
} // namespace memgraph::query::v2

View File

@ -0,0 +1,19 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/bindings/bindings.hpp"
#include "expr/ast/pretty_print.hpp"
namespace memgraph::query::v2 {
} // namespace memgraph::query::v2

View File

@ -0,0 +1,20 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/bindings/bindings.hpp"
#include "expr/semantic/symbol.hpp"
namespace memgraph::query::v2 {
using Symbol = memgraph::expr::Symbol;
} // namespace memgraph::query::v2

View File

@ -0,0 +1,16 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/bindings/bindings.hpp"
#include "expr/semantic/symbol_generator.hpp"

View File

@ -0,0 +1,20 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/bindings/bindings.hpp"
#include "expr/semantic/symbol_table.hpp"
namespace memgraph::query::v2 {
using SymbolTable = memgraph::expr::SymbolTable;
} // namespace memgraph::query::v2

View File

@ -0,0 +1,19 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "expr/typed_value.hpp"
#include "query/v2/accessors.hpp"
#include "query/v2/path.hpp"
namespace memgraph::expr {
namespace v2 = memgraph::query::v2;
template class TypedValueT<v2::accessors::VertexAccessor, v2::accessors::EdgeAccessor, v2::accessors::Path>;
} // namespace memgraph::expr

View File

@ -0,0 +1,27 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/bindings/bindings.hpp"
#include "expr/typed_value.hpp"
#include "query/v2/accessors.hpp"
namespace memgraph::expr {
namespace v2 = memgraph::query::v2;
extern template class memgraph::expr::TypedValueT<v2::accessors::VertexAccessor, v2::accessors::EdgeAccessor,
v2::accessors::Path>;
} // namespace memgraph::expr
namespace memgraph::query::v2 {
using TypedValue =
memgraph::expr::TypedValueT<v2::accessors::VertexAccessor, v2::accessors::EdgeAccessor, v2::accessors::Path>;
} // namespace memgraph::query::v2

76
src/query/v2/common.cpp Normal file
View File

@ -0,0 +1,76 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/v2/common.hpp"
namespace memgraph::query::v2 {
namespace impl {
bool TypedValueCompare(const TypedValue &a, const TypedValue &b) {
// in ordering null comes after everything else
// at the same time Null is not less that null
// first deal with Null < Whatever case
if (a.IsNull()) return false;
// now deal with NotNull < Null case
if (b.IsNull()) return true;
// comparisons are from this point legal only between values of
// the same type, or int+float combinations
if ((a.type() != b.type() && !(a.IsNumeric() && b.IsNumeric())))
throw QueryRuntimeException("Can't compare value of type {} to value of type {}.", a.type(), b.type());
switch (a.type()) {
case TypedValue::Type::Bool:
return !a.ValueBool() && b.ValueBool();
case TypedValue::Type::Int:
if (b.type() == TypedValue::Type::Double)
return a.ValueInt() < b.ValueDouble();
else
return a.ValueInt() < b.ValueInt();
case TypedValue::Type::Double:
if (b.type() == TypedValue::Type::Int)
return a.ValueDouble() < b.ValueInt();
else
return a.ValueDouble() < b.ValueDouble();
case TypedValue::Type::String:
// NOLINTNEXTLINE(modernize-use-nullptr)
return a.ValueString() < b.ValueString();
case TypedValue::Type::Date:
// NOLINTNEXTLINE(modernize-use-nullptr)
return a.ValueDate() < b.ValueDate();
case TypedValue::Type::LocalTime:
// NOLINTNEXTLINE(modernize-use-nullptr)
return a.ValueLocalTime() < b.ValueLocalTime();
case TypedValue::Type::LocalDateTime:
// NOLINTNEXTLINE(modernize-use-nullptr)
return a.ValueLocalDateTime() < b.ValueLocalDateTime();
case TypedValue::Type::Duration:
// NOLINTNEXTLINE(modernize-use-nullptr)
return a.ValueDuration() < b.ValueDuration();
case TypedValue::Type::List:
case TypedValue::Type::Map:
case TypedValue::Type::Vertex:
case TypedValue::Type::Edge:
case TypedValue::Type::Path:
throw QueryRuntimeException("Comparison is not defined for values of type {}.", a.type());
case TypedValue::Type::Null:
LOG_FATAL("Invalid type");
}
}
} // namespace impl
int64_t QueryTimestamp() {
return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch())
.count();
}
} // namespace memgraph::query::v2

183
src/query/v2/common.hpp Normal file
View File

@ -0,0 +1,183 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file
#pragma once
#include <concepts>
#include <cstdint>
#include <string>
#include <string_view>
#include <type_traits>
#include "query/v2/bindings/symbol.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/db_accessor.hpp"
#include "query/v2/exceptions.hpp"
#include "query/v2/frontend/ast/ast.hpp"
#include "query/v2/path.hpp"
#include "storage/v3/conversions.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/view.hpp"
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
#include "utils/variant_helpers.hpp"
namespace memgraph::query::v2 {
namespace impl {
bool TypedValueCompare(const TypedValue &a, const TypedValue &b);
} // namespace impl
/// Custom Comparator type for comparing vectors of TypedValues.
///
/// Does lexicographical ordering of elements based on the above
/// defined TypedValueCompare, and also accepts a vector of Orderings
/// the define how respective elements compare.
class TypedValueVectorCompare final {
public:
TypedValueVectorCompare() {}
explicit TypedValueVectorCompare(const std::vector<Ordering> &ordering) : ordering_(ordering) {}
template <class TAllocator>
bool operator()(const std::vector<TypedValue, TAllocator> &c1, const std::vector<TypedValue, TAllocator> &c2) const {
// ordering is invalid if there are more elements in the collections
// then there are in the ordering_ vector
MG_ASSERT(c1.size() <= ordering_.size() && c2.size() <= ordering_.size(),
"Collections contain more elements then there are orderings");
auto c1_it = c1.begin();
auto c2_it = c2.begin();
auto ordering_it = ordering_.begin();
for (; c1_it != c1.end() && c2_it != c2.end(); c1_it++, c2_it++, ordering_it++) {
if (impl::TypedValueCompare(*c1_it, *c2_it)) return *ordering_it == Ordering::ASC;
if (impl::TypedValueCompare(*c2_it, *c1_it)) return *ordering_it == Ordering::DESC;
}
// at least one collection is exhausted
// c1 is less then c2 iff c1 reached the end but c2 didn't
return (c1_it == c1.end()) && (c2_it != c2.end());
}
// TODO: Remove this, member is public
const auto &ordering() const { return ordering_; }
std::vector<Ordering> ordering_;
};
/// Raise QueryRuntimeException if the value for symbol isn't of expected type.
inline void ExpectType(const Symbol &symbol, const TypedValue &value, TypedValue::Type expected) {
if (value.type() != expected)
throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected, symbol.name(), value.type());
}
template <typename T>
concept AccessorWithSetProperty = requires(T accessor, const storage::v3::PropertyId key,
const storage::v3::PropertyValue new_value) {
{ accessor.SetProperty(key, new_value) } -> std::same_as<storage::v3::Result<storage::v3::PropertyValue>>;
};
template <typename T>
concept AccessorWithSetPropertyAndValidate = requires(T accessor, const storage::v3::PropertyId key,
const storage::v3::PropertyValue new_value) {
{
accessor.SetPropertyAndValidate(key, new_value)
} -> std::same_as<storage::v3::ResultSchema<storage::v3::PropertyValue>>;
};
template <typename TRecordAccessor>
concept RecordAccessor =
AccessorWithSetProperty<TRecordAccessor> || AccessorWithSetPropertyAndValidate<TRecordAccessor>;
inline void HandleSchemaViolation(const storage::v3::SchemaViolation &schema_violation, const DbAccessor &dba) {
switch (schema_violation.status) {
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_HAS_NO_PRIMARY_PROPERTY: {
throw SchemaViolationException(
fmt::format("Primary key {} not defined on label :{}",
storage::v3::SchemaTypeToString(schema_violation.violated_schema_property->type),
dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::NO_SCHEMA_DEFINED_FOR_LABEL: {
throw SchemaViolationException(
fmt::format("Label :{} is not a primary label", dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_PROPERTY_WRONG_TYPE: {
throw SchemaViolationException(
fmt::format("Wrong type of property {} in schema :{}, should be of type {}",
*schema_violation.violated_property_value, dba.LabelToName(schema_violation.label),
storage::v3::SchemaTypeToString(schema_violation.violated_schema_property->type)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_KEY: {
throw SchemaViolationException(fmt::format("Updating of primary key {} on schema :{} not supported",
*schema_violation.violated_property_value,
dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL: {
throw SchemaViolationException(fmt::format(
"Adding primary label as secondary or removing primary label:", *schema_violation.violated_property_value,
dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY: {
throw SchemaViolationException(fmt::format("Cannot create vertex where primary label is secondary:{}",
dba.LabelToName(schema_violation.label)));
}
}
}
inline void HandleErrorOnPropertyUpdate(const storage::v3::Error error) {
switch (error) {
case storage::v3::Error::SERIALIZATION_ERROR:
throw TransactionSerializationException();
case storage::v3::Error::DELETED_OBJECT:
throw QueryRuntimeException("Trying to set properties on a deleted object.");
case storage::v3::Error::PROPERTIES_DISABLED:
throw QueryRuntimeException("Can't set property because properties on edges are disabled.");
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::NONEXISTENT_OBJECT:
throw QueryRuntimeException("Unexpected error when setting a property.");
}
}
/// Set a property `value` mapped with given `key` on a `record`.
///
/// @throw QueryRuntimeException if value cannot be set as a property value
template <RecordAccessor T>
storage::v3::PropertyValue PropsSetChecked(T *record, const DbAccessor &dba, const storage::v3::PropertyId &key,
const TypedValue &value) {
try {
if constexpr (std::is_same_v<T, VertexAccessor>) {
const auto maybe_old_value = record->SetPropertyAndValidate(key, storage::v3::TypedToPropertyValue(value));
if (maybe_old_value.HasError()) {
std::visit(utils::Overloaded{[](const storage::v3::Error error) { HandleErrorOnPropertyUpdate(error); },
[&dba](const storage::v3::SchemaViolation &schema_violation) {
HandleSchemaViolation(schema_violation, dba);
}},
maybe_old_value.GetError());
}
return std::move(*maybe_old_value);
} else {
// No validation on edge properties
const auto maybe_old_value = record->SetProperty(key, storage::v3::TypedToPropertyValue(value));
if (maybe_old_value.HasError()) {
HandleErrorOnPropertyUpdate(maybe_old_value.GetError());
}
return std::move(*maybe_old_value);
}
} catch (const expr::TypedValueException &) {
throw QueryRuntimeException("'{}' cannot be used as a property value.", value.type());
}
}
int64_t QueryTimestamp();
} // namespace memgraph::query::v2

32
src/query/v2/config.hpp Normal file
View File

@ -0,0 +1,32 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <string>
namespace memgraph::query::v2 {
struct InterpreterConfig {
struct Query {
bool allow_load_csv{true};
} query;
// The default execution timeout is 10 minutes.
double execution_timeout_sec{600.0};
// The same as \ref memgraph::storage::v3::replication::ReplicationClientConfig
std::chrono::seconds replication_replica_check_frequency{1};
std::string default_kafka_bootstrap_servers;
std::string default_pulsar_service_url;
uint32_t stream_transaction_conflict_retries;
std::chrono::milliseconds stream_transaction_retry_interval;
};
} // namespace memgraph::query::v2

View File

@ -0,0 +1,19 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cstdint>
#include <string>
namespace memgraph::query::v2 {
inline constexpr uint16_t kDefaultReplicationPort = 10000;
inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
} // namespace memgraph::query::v2

91
src/query/v2/context.hpp Normal file
View File

@ -0,0 +1,91 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <type_traits>
#include "query/v2/bindings/symbol_table.hpp"
#include "query/v2/common.hpp"
#include "query/v2/metadata.hpp"
#include "query/v2/parameters.hpp"
#include "query/v2/plan/profile.hpp"
//#include "query/v2/trigger.hpp"
#include "query/v2/shard_request_manager.hpp"
#include "utils/async_timer.hpp"
namespace memgraph::query::v2 {
struct EvaluationContext {
/// Memory for allocations during evaluation of a *single* Pull call.
///
/// Although the assigned memory may live longer than the duration of a Pull
/// (e.g. memory is the same as the whole execution memory), you have to treat
/// it as if the lifetime is only valid during the Pull.
utils::MemoryResource *memory{utils::NewDeleteResource()};
int64_t timestamp{-1};
Parameters parameters;
/// All properties indexable via PropertyIx
std::vector<storage::v3::PropertyId> properties;
/// All labels indexable via LabelIx
std::vector<storage::v3::LabelId> labels;
/// All counters generated by `counter` function, mutable because the function
/// modifies the values
mutable std::unordered_map<std::string, int64_t> counters;
};
inline std::vector<storage::v3::PropertyId> NamesToProperties(const std::vector<std::string> &property_names,
DbAccessor *dba) {
std::vector<storage::v3::PropertyId> properties;
properties.reserve(property_names.size());
for (const auto &name : property_names) {
properties.push_back(dba->NameToProperty(name));
}
return properties;
}
inline std::vector<storage::v3::LabelId> NamesToLabels(const std::vector<std::string> &label_names, DbAccessor *dba) {
std::vector<storage::v3::LabelId> labels;
labels.reserve(label_names.size());
for (const auto &name : label_names) {
labels.push_back(dba->NameToLabel(name));
}
return labels;
}
struct ExecutionContext {
DbAccessor *db_accessor{nullptr};
SymbolTable symbol_table;
EvaluationContext evaluation_context;
std::atomic<bool> *is_shutting_down{nullptr};
bool is_profile_query{false};
std::chrono::duration<double> profile_execution_time;
plan::ProfilingStats stats;
plan::ProfilingStats *stats_root{nullptr};
ExecutionStats execution_stats;
// TriggerContextCollector *trigger_context_collector{nullptr};
utils::AsyncTimer timer;
std::unique_ptr<msgs::ShardRequestManagerInterface> shard_request_manager{nullptr};
};
static_assert(std::is_move_assignable_v<ExecutionContext>, "ExecutionContext must be move assignable!");
static_assert(std::is_move_constructible_v<ExecutionContext>, "ExecutionContext must be move constructible!");
inline bool MustAbort(const ExecutionContext &context) noexcept {
return (context.is_shutting_down != nullptr && context.is_shutting_down->load(std::memory_order_acquire)) ||
context.timer.IsExpired();
}
inline plan::ProfilingStatsWithTotalTime GetStatsWithTotalTime(const ExecutionContext &context) {
return plan::ProfilingStatsWithTotalTime{context.stats, context.profile_execution_time};
}
} // namespace memgraph::query::v2

View File

@ -0,0 +1,100 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "bindings/typed_value.hpp"
#include "query/v2/accessors.hpp"
#include "query/v2/requests.hpp"
namespace memgraph::query::v2 {
inline TypedValue ValueToTypedValue(const msgs::Value &value) {
using Value = msgs::Value;
switch (value.type) {
case Value::Type::Null:
return {};
case Value::Type::Bool:
return TypedValue(value.bool_v);
case Value::Type::Int64:
return TypedValue(value.int_v);
case Value::Type::Double:
return TypedValue(value.double_v);
case Value::Type::String:
return TypedValue(value.string_v);
case Value::Type::List: {
const auto &lst = value.list_v;
std::vector<TypedValue> dst;
dst.reserve(lst.size());
for (const auto &elem : lst) {
dst.push_back(ValueToTypedValue(elem));
}
return TypedValue(std::move(dst));
}
case Value::Type::Map: {
const auto &value_map = value.map_v;
std::map<std::string, TypedValue> dst;
for (const auto &[key, val] : value_map) {
dst[key] = ValueToTypedValue(val);
}
return TypedValue(std::move(dst));
}
case Value::Type::Vertex:
return TypedValue(accessors::VertexAccessor(value.vertex_v, {}));
case Value::Type::Edge:
return TypedValue(accessors::EdgeAccessor(value.edge_v, {}));
case Value::Type::Path:
break;
}
throw std::runtime_error("Incorrect type in conversion");
}
inline msgs::Value TypedValueToValue(const TypedValue &value) {
using Value = msgs::Value;
switch (value.type()) {
case TypedValue::Type::Null:
return {};
case TypedValue::Type::Bool:
return Value(value.ValueBool());
case TypedValue::Type::Int:
return Value(value.ValueInt());
case TypedValue::Type::Double:
return Value(value.ValueDouble());
case TypedValue::Type::String:
return Value(std::string(value.ValueString()));
case TypedValue::Type::List: {
const auto &lst = value.ValueList();
std::vector<Value> dst;
dst.reserve(lst.size());
for (const auto &elem : lst) {
dst.push_back(TypedValueToValue(elem));
}
return Value(std::move(dst));
}
case TypedValue::Type::Map: {
const auto &value_map = value.ValueMap();
std::map<std::string, Value> dst;
for (const auto &[key, val] : value_map) {
dst[std::string(key)] = TypedValueToValue(val);
}
return Value(std::move(dst));
}
case TypedValue::Type::Vertex:
return Value(value.ValueVertex().GetVertex());
case TypedValue::Type::Edge:
return Value(value.ValueEdge().GetEdge());
case TypedValue::Type::Path:
default:
break;
}
throw std::runtime_error("Incorrect type in conversion");
}
} // namespace memgraph::query::v2

View File

@ -0,0 +1,154 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/v2/cypher_query_interpreter.hpp"
#include "query/v2/bindings/symbol_generator.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_bool(query_cost_planner, true, "Use the cost-estimating query planner.");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(query_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
FLAG_IN_RANGE(0, std::numeric_limits<int32_t>::max()));
namespace memgraph::query::v2 {
CachedPlan::CachedPlan(std::unique_ptr<LogicalPlan> plan) : plan_(std::move(plan)) {}
ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::string, storage::v3::PropertyValue> &params,
utils::SkipList<QueryCacheEntry> *cache, const InterpreterConfig::Query &query_config) {
// Strip the query for caching purposes. The process of stripping a query
// "normalizes" it by replacing any literals with new parameters. This
// results in just the *structure* of the query being taken into account for
// caching.
frontend::StrippedQuery stripped_query{query_string};
// Copy over the parameters that were introduced during stripping.
Parameters parameters{stripped_query.literals()};
// Check that all user-specified parameters are provided.
for (const auto &param_pair : stripped_query.parameters()) {
auto it = params.find(param_pair.second);
if (it == params.end()) {
throw query::v2::UnprovidedParameterError("Parameter ${} not provided.", param_pair.second);
}
parameters.Add(param_pair.first, it->second);
}
// Cache the query's AST if it isn't already.
auto hash = stripped_query.hash();
auto accessor = cache->access();
auto it = accessor.find(hash);
std::unique_ptr<memgraph::frontend::opencypher::Parser<>> parser;
// Return a copy of both the AST storage and the query.
CachedQuery result;
bool is_cacheable = true;
auto get_information_from_cache = [&](const auto &cached_query) {
result.ast_storage.properties_ = cached_query.ast_storage.properties_;
result.ast_storage.labels_ = cached_query.ast_storage.labels_;
result.ast_storage.edge_types_ = cached_query.ast_storage.edge_types_;
result.query = cached_query.query->Clone(&result.ast_storage);
result.required_privileges = cached_query.required_privileges;
};
if (it == accessor.end()) {
try {
parser = std::make_unique<memgraph::frontend::opencypher::Parser<>>(stripped_query.query());
} catch (const SyntaxException &e) {
// There is a syntax exception in the stripped query. Re-run the parser
// on the original query to get an appropriate error messsage.
parser = std::make_unique<memgraph::frontend::opencypher::Parser<>>(query_string);
// If an exception was not thrown here, the stripper messed something
// up.
LOG_FATAL("The stripped query can't be parsed, but the original can.");
}
// Convert the ANTLR4 parse tree into an AST.
AstStorage ast_storage;
expr::ParsingContext context{true};
memgraph::expr::CypherMainVisitor visitor(context, &ast_storage);
visitor.visit(parser->tree());
if (visitor.GetQueryInfo().has_load_csv && !query_config.allow_load_csv) {
throw utils::BasicException("Load CSV not allowed on this instance because it was disabled by a config.");
}
if (visitor.GetQueryInfo().is_cacheable) {
CachedQuery cached_query{std::move(ast_storage), visitor.query(),
query::v2::GetRequiredPrivileges(visitor.query())};
it = accessor.insert({hash, std::move(cached_query)}).first;
get_information_from_cache(it->second);
} else {
result.ast_storage.properties_ = ast_storage.properties_;
result.ast_storage.labels_ = ast_storage.labels_;
result.ast_storage.edge_types_ = ast_storage.edge_types_;
result.query = visitor.query()->Clone(&result.ast_storage);
result.required_privileges = query::v2::GetRequiredPrivileges(visitor.query());
is_cacheable = false;
}
} else {
get_information_from_cache(it->second);
}
return ParsedQuery{query_string,
params,
std::move(parameters),
std::move(stripped_query),
std::move(result.ast_storage),
result.query,
std::move(result.required_privileges),
is_cacheable};
}
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers) {
auto vertex_counts = plan::MakeVertexCountCache(db_accessor);
auto symbol_table = expr::MakeSymbolTable(query, predefined_identifiers);
auto planning_context = plan::MakePlanningContext(&ast_storage, &symbol_table, query, &vertex_counts);
auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_cost_planner);
return std::make_unique<SingleNodeLogicalPlan>(std::move(root), cost, std::move(ast_storage),
std::move(symbol_table));
}
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers) {
std::optional<utils::SkipList<PlanCacheEntry>::Accessor> plan_cache_access;
if (plan_cache) {
plan_cache_access.emplace(plan_cache->access());
auto it = plan_cache_access->find(hash);
if (it != plan_cache_access->end()) {
if (it->second->IsExpired()) {
plan_cache_access->remove(hash);
} else {
return it->second;
}
}
}
auto plan = std::make_shared<CachedPlan>(
MakeLogicalPlan(std::move(ast_storage), query, parameters, db_accessor, predefined_identifiers));
if (plan_cache_access) {
plan_cache_access->insert({hash, plan});
}
return plan;
}
} // namespace memgraph::query::v2

View File

@ -0,0 +1,151 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "parser/opencypher/parser.hpp"
#include "query/v2/bindings/cypher_main_visitor.hpp"
#include "query/v2/bindings/symbol_table.hpp"
#include "query/v2/config.hpp"
#include "query/v2/frontend/semantic/required_privileges.hpp"
#include "query/v2/frontend/stripped.hpp"
#include "query/v2/plan/planner.hpp"
#include "utils/flag_validation.hpp"
#include "utils/timer.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(query_cost_planner);
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_int32(query_plan_cache_ttl);
namespace memgraph::query::v2 {
// TODO: Maybe this should move to query/plan/planner.
/// Interface for accessing the root operator of a logical plan.
class LogicalPlan {
public:
explicit LogicalPlan() = default;
virtual ~LogicalPlan() = default;
LogicalPlan(const LogicalPlan &) = default;
LogicalPlan &operator=(const LogicalPlan &) = default;
LogicalPlan(LogicalPlan &&) = default;
LogicalPlan &operator=(LogicalPlan &&) = default;
virtual const plan::LogicalOperator &GetRoot() const = 0;
virtual double GetCost() const = 0;
virtual const SymbolTable &GetSymbolTable() const = 0;
virtual const AstStorage &GetAstStorage() const = 0;
};
class CachedPlan {
public:
explicit CachedPlan(std::unique_ptr<LogicalPlan> plan);
const auto &plan() const { return plan_->GetRoot(); }
double cost() const { return plan_->GetCost(); }
const auto &symbol_table() const { return plan_->GetSymbolTable(); }
const auto &ast_storage() const { return plan_->GetAstStorage(); }
bool IsExpired() const {
// NOLINTNEXTLINE (modernize-use-nullptr)
return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_plan_cache_ttl);
};
private:
std::unique_ptr<LogicalPlan> plan_;
utils::Timer cache_timer_;
};
struct CachedQuery {
AstStorage ast_storage;
Query *query;
std::vector<AuthQuery::Privilege> required_privileges;
};
struct QueryCacheEntry {
bool operator==(const QueryCacheEntry &other) const { return first == other.first; }
bool operator<(const QueryCacheEntry &other) const { return first < other.first; }
bool operator==(const uint64_t &other) const { return first == other; }
bool operator<(const uint64_t &other) const { return first < other; }
uint64_t first;
// TODO: Maybe store the query string here and use it as a key with the hash
// so that we eliminate the risk of hash collisions.
CachedQuery second;
};
struct PlanCacheEntry {
bool operator==(const PlanCacheEntry &other) const { return first == other.first; }
bool operator<(const PlanCacheEntry &other) const { return first < other.first; }
bool operator==(const uint64_t &other) const { return first == other; }
bool operator<(const uint64_t &other) const { return first < other; }
uint64_t first;
// TODO: Maybe store the query string here and use it as a key with the hash
// so that we eliminate the risk of hash collisions.
std::shared_ptr<CachedPlan> second;
};
/**
* A container for data related to the parsing of a query.
*/
struct ParsedQuery {
std::string query_string;
std::map<std::string, storage::v3::PropertyValue> user_parameters;
Parameters parameters;
frontend::StrippedQuery stripped_query;
AstStorage ast_storage;
Query *query;
std::vector<AuthQuery::Privilege> required_privileges;
bool is_cacheable{true};
};
ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::string, storage::v3::PropertyValue> &params,
utils::SkipList<QueryCacheEntry> *cache, const InterpreterConfig::Query &query_config);
class SingleNodeLogicalPlan final : public LogicalPlan {
public:
SingleNodeLogicalPlan(std::unique_ptr<plan::LogicalOperator> root, double cost, AstStorage storage,
const SymbolTable &symbol_table)
: root_(std::move(root)), cost_(cost), storage_(std::move(storage)), symbol_table_(symbol_table) {}
const plan::LogicalOperator &GetRoot() const override { return *root_; }
double GetCost() const override { return cost_; }
const SymbolTable &GetSymbolTable() const override { return symbol_table_; }
const AstStorage &GetAstStorage() const override { return storage_; }
private:
std::unique_ptr<plan::LogicalOperator> root_;
double cost_;
AstStorage storage_;
SymbolTable symbol_table_;
};
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers);
/**
* Return the parsed *Cypher* query's AST cached logical plan, or create and
* cache a fresh one if it doesn't yet exist.
* @param predefined_identifiers optional identifiers you want to inject into a query.
* If an identifier is not defined in a scope, we check the predefined identifiers.
* If an identifier is contained there, we inject it at that place and remove it,
* because a predefined identifier can be used only in one scope.
*/
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers = {});
} // namespace memgraph::query::v2

View File

@ -0,0 +1,431 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cstdint>
#include <optional>
#include <vector>
#include <cppitertools/filter.hpp>
#include <cppitertools/imap.hpp>
#include "query/v2/exceptions.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
///////////////////////////////////////////////////////////
// Our communication layer and query engine don't mix
// very well on Centos because OpenSSL version available
// on Centos 7 include libkrb5 which has brilliant macros
// called TRUE and FALSE. For more detailed explanation go
// to memgraph.cpp.
//
// Because of the replication storage now uses some form of
// communication so we have some unwanted macros.
// This cannot be avoided by simple include orderings so we
// simply undefine those macros as we're sure that libkrb5
// won't and can't be used anywhere in the query engine.
#include "storage/v3/storage.hpp"
#include "utils/logging.hpp"
#include "utils/result.hpp"
#undef FALSE
#undef TRUE
///////////////////////////////////////////////////////////
#include "storage/v3/view.hpp"
#include "utils/bound.hpp"
#include "utils/exceptions.hpp"
namespace memgraph::query::v2 {
class VertexAccessor;
class EdgeAccessor final {
public:
storage::v3::EdgeAccessor impl_;
explicit EdgeAccessor(storage::v3::EdgeAccessor impl) : impl_(std::move(impl)) {}
bool IsVisible(storage::v3::View view) const { return impl_.IsVisible(view); }
storage::v3::EdgeTypeId EdgeType() const { return impl_.EdgeType(); }
auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
storage::v3::PropertyId key) const {
return impl_.GetProperty(key, view);
}
storage::v3::Result<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
const storage::v3::PropertyValue &value) {
return impl_.SetProperty(key, value);
}
storage::v3::Result<storage::v3::PropertyValue> RemoveProperty(storage::v3::PropertyId key) {
return SetProperty(key, storage::v3::PropertyValue());
}
storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
return impl_.ClearProperties();
}
VertexAccessor To() const;
VertexAccessor From() const;
bool IsCycle() const;
int64_t CypherId() const { return impl_.Gid().AsInt(); }
storage::v3::Gid Gid() const noexcept { return impl_.Gid(); }
bool operator==(const EdgeAccessor &e) const noexcept { return impl_ == e.impl_; }
bool operator!=(const EdgeAccessor &e) const noexcept { return !(*this == e); }
};
class VertexAccessor final {
public:
storage::v3::VertexAccessor impl_;
static EdgeAccessor MakeEdgeAccessor(const storage::v3::EdgeAccessor impl) { return EdgeAccessor(impl); }
explicit VertexAccessor(storage::v3::VertexAccessor impl) : impl_(impl) {}
bool IsVisible(storage::v3::View view) const { return impl_.IsVisible(view); }
auto Labels(storage::v3::View view) const { return impl_.Labels(view); }
auto PrimaryLabel(storage::v3::View view) const { return impl_.PrimaryLabel(view); }
auto PrimaryKey(storage::v3::View view) const { return impl_.PrimaryKey(view); }
storage::v3::ResultSchema<bool> AddLabel(storage::v3::LabelId label) { return impl_.AddLabelAndValidate(label); }
storage::v3::ResultSchema<bool> AddLabelAndValidate(storage::v3::LabelId label) {
return impl_.AddLabelAndValidate(label);
}
storage::v3::ResultSchema<bool> RemoveLabel(storage::v3::LabelId label) {
return impl_.RemoveLabelAndValidate(label);
}
storage::v3::ResultSchema<bool> RemoveLabelAndValidate(storage::v3::LabelId label) {
return impl_.RemoveLabelAndValidate(label);
}
storage::v3::Result<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
return impl_.HasLabel(label, view);
}
auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
storage::v3::PropertyId key) const {
return impl_.GetProperty(key, view);
}
storage::v3::ResultSchema<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
const storage::v3::PropertyValue &value) {
return impl_.SetPropertyAndValidate(key, value);
}
storage::v3::ResultSchema<storage::v3::PropertyValue> SetPropertyAndValidate(
storage::v3::PropertyId key, const storage::v3::PropertyValue &value) {
return impl_.SetPropertyAndValidate(key, value);
}
storage::v3::ResultSchema<storage::v3::PropertyValue> RemovePropertyAndValidate(storage::v3::PropertyId key) {
return SetPropertyAndValidate(key, storage::v3::PropertyValue{});
}
storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
return impl_.ClearProperties();
}
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
auto maybe_edges = impl_.InEdges(view, edge_types);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
auto InEdges(storage::v3::View view) const { return InEdges(view, {}); }
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
const VertexAccessor &dest) const
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
const auto dest_id = dest.impl_.Id(view).GetValue();
auto maybe_edges = impl_.InEdges(view, edge_types, &dest_id);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
auto maybe_edges = impl_.OutEdges(view, edge_types);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
auto OutEdges(storage::v3::View view) const { return OutEdges(view, {}); }
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
const VertexAccessor &dest) const
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
const auto dest_id = dest.impl_.Id(view).GetValue();
auto maybe_edges = impl_.OutEdges(view, edge_types, &dest_id);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
storage::v3::Result<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
storage::v3::Result<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
// TODO(jbajic) Fix Remove Gid
static int64_t CypherId() { return 1; }
bool operator==(const VertexAccessor &v) const noexcept {
static_assert(noexcept(impl_ == v.impl_));
return impl_ == v.impl_;
}
bool operator!=(const VertexAccessor &v) const noexcept { return !(*this == v); }
};
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wnull-dereference"
// NOLINTNEXTLINE(readability-convert-member-functions-to-static,clang-analyzer-core.NonNullParamChecker)
inline VertexAccessor EdgeAccessor::To() const { return *static_cast<VertexAccessor *>(nullptr); }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static,clang-analyzer-core.NonNullParamChecker)
inline VertexAccessor EdgeAccessor::From() const { return *static_cast<VertexAccessor *>(nullptr); }
#pragma clang diagnostic pop
inline bool EdgeAccessor::IsCycle() const { return To() == From(); }
class DbAccessor final {
storage::v3::Shard::Accessor *accessor_;
class VerticesIterable final {
storage::v3::VerticesIterable iterable_;
public:
class Iterator final {
storage::v3::VerticesIterable::Iterator it_;
public:
explicit Iterator(storage::v3::VerticesIterable::Iterator it) : it_(it) {}
VertexAccessor operator*() const { return VertexAccessor(*it_); }
Iterator &operator++() {
++it_;
return *this;
}
bool operator==(const Iterator &other) const { return it_ == other.it_; }
bool operator!=(const Iterator &other) const { return !(other == *this); }
};
explicit VerticesIterable(storage::v3::VerticesIterable iterable) : iterable_(std::move(iterable)) {}
Iterator begin() { return Iterator(iterable_.begin()); }
Iterator end() { return Iterator(iterable_.end()); }
};
public:
explicit DbAccessor(storage::v3::Shard::Accessor *accessor) : accessor_(accessor) {}
// TODO(jbajic) Fix Remove Gid
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
std::optional<VertexAccessor> FindVertex(uint64_t /*unused*/) { return std::nullopt; }
std::optional<VertexAccessor> FindVertex(storage::v3::PrimaryKey &primary_key, storage::v3::View view) {
auto maybe_vertex = accessor_->FindVertex(primary_key, view);
if (maybe_vertex) return VertexAccessor(*maybe_vertex);
return std::nullopt;
}
VerticesIterable Vertices(storage::v3::View view) { return VerticesIterable(accessor_->Vertices(view)); }
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label) {
return VerticesIterable(accessor_->Vertices(label, view));
}
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property) {
return VerticesIterable(accessor_->Vertices(label, property, view));
}
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property,
const storage::v3::PropertyValue &value) {
return VerticesIterable(accessor_->Vertices(label, property, value, view));
}
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property,
const std::optional<utils::Bound<storage::v3::PropertyValue>> &lower,
const std::optional<utils::Bound<storage::v3::PropertyValue>> &upper) {
return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view));
}
storage::v3::ResultSchema<VertexAccessor> InsertVertexAndValidate(
const storage::v3::LabelId primary_label, const std::vector<storage::v3::LabelId> &labels,
const std::vector<std::pair<storage::v3::PropertyId, storage::v3::PropertyValue>> &properties) {
auto maybe_vertex_acc = accessor_->CreateVertexAndValidate(primary_label, labels, properties);
if (maybe_vertex_acc.HasError()) {
return {std::move(maybe_vertex_acc.GetError())};
}
return VertexAccessor{maybe_vertex_acc.GetValue()};
}
storage::v3::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
const storage::v3::EdgeTypeId &edge_type) {
static constexpr auto kDummyGid = storage::v3::Gid::FromUint(0);
auto maybe_edge = accessor_->CreateEdge(from->impl_.Id(storage::v3::View::NEW).GetValue(),
to->impl_.Id(storage::v3::View::NEW).GetValue(), edge_type, kDummyGid);
if (maybe_edge.HasError()) return storage::v3::Result<EdgeAccessor>(maybe_edge.GetError());
return EdgeAccessor(*maybe_edge);
}
storage::v3::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
auto res = accessor_->DeleteEdge(edge->impl_.FromVertex(), edge->impl_.ToVertex(), edge->impl_.Gid());
if (res.HasError()) {
return res.GetError();
}
const auto &value = res.GetValue();
if (!value) {
return std::optional<EdgeAccessor>{};
}
return std::make_optional<EdgeAccessor>(*value);
}
storage::v3::Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
VertexAccessor *vertex_accessor) {
using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
auto res = accessor_->DetachDeleteVertex(&vertex_accessor->impl_);
if (res.HasError()) {
return res.GetError();
}
const auto &value = res.GetValue();
if (!value) {
return std::optional<ReturnType>{};
}
const auto &[vertex, edges] = *value;
std::vector<EdgeAccessor> deleted_edges;
deleted_edges.reserve(edges.size());
std::transform(edges.begin(), edges.end(), std::back_inserter(deleted_edges),
[](const auto &deleted_edge) { return EdgeAccessor{deleted_edge}; });
return std::make_optional<ReturnType>(vertex, std::move(deleted_edges));
}
storage::v3::Result<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
auto res = accessor_->DeleteVertex(&vertex_accessor->impl_);
if (res.HasError()) {
return res.GetError();
}
const auto &value = res.GetValue();
if (!value) {
return std::optional<VertexAccessor>{};
}
return {std::make_optional<VertexAccessor>(*value)};
}
// TODO(jbajic) Query engine should have a map of labels, properties and edge
// types
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
storage::v3::PropertyId NameToProperty(const std::string_view name) { return accessor_->NameToProperty(name); }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
storage::v3::LabelId NameToLabel(const std::string_view name) { return accessor_->NameToLabel(name); }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
storage::v3::EdgeTypeId NameToEdgeType(const std::string_view name) { return accessor_->NameToEdgeType(name); }
const std::string &PropertyToName(storage::v3::PropertyId prop) const { return accessor_->PropertyToName(prop); }
const std::string &LabelToName(storage::v3::LabelId label) const { return accessor_->LabelToName(label); }
const std::string &EdgeTypeToName(storage::v3::EdgeTypeId type) const { return accessor_->EdgeTypeToName(type); }
void AdvanceCommand() { accessor_->AdvanceCommand(); }
void Commit() { return accessor_->Commit(coordinator::Hlc{}); }
void Abort() { accessor_->Abort(); }
bool LabelIndexExists(storage::v3::LabelId label) const { return accessor_->LabelIndexExists(label); }
bool LabelPropertyIndexExists(storage::v3::LabelId label, storage::v3::PropertyId prop) const {
return accessor_->LabelPropertyIndexExists(label, prop);
}
int64_t VerticesCount() const { return accessor_->ApproximateVertexCount(); }
int64_t VerticesCount(storage::v3::LabelId label) const { return accessor_->ApproximateVertexCount(label); }
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property) const {
return accessor_->ApproximateVertexCount(label, property);
}
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property,
const storage::v3::PropertyValue &value) const {
return accessor_->ApproximateVertexCount(label, property, value);
}
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property,
const std::optional<utils::Bound<storage::v3::PropertyValue>> &lower,
const std::optional<utils::Bound<storage::v3::PropertyValue>> &upper) const {
return accessor_->ApproximateVertexCount(label, property, lower, upper);
}
storage::v3::IndicesInfo ListAllIndices() const { return accessor_->ListAllIndices(); }
const storage::v3::SchemaValidator &GetSchemaValidator() const { return accessor_->GetSchemaValidator(); }
storage::v3::SchemasInfo ListAllSchemas() const { return accessor_->ListAllSchemas(); }
};
} // namespace memgraph::query::v2
namespace std {
template <>
struct hash<memgraph::query::v2::VertexAccessor> {
size_t operator()(const memgraph::query::v2::VertexAccessor &v) const {
return std::hash<decltype(v.impl_)>{}(v.impl_);
}
};
template <>
struct hash<memgraph::query::v2::EdgeAccessor> {
size_t operator()(const memgraph::query::v2::EdgeAccessor &e) const {
return std::hash<decltype(e.impl_)>{}(e.impl_);
}
};
} // namespace std

View File

@ -0,0 +1,24 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <vector>
#include "query/v2/bindings/typed_value.hpp"
namespace memgraph::query::v2 {
struct DiscardValueResultStream final {
void Result(const std::vector<query::v2::TypedValue> & /*values*/) {
// do nothing
}
};
} // namespace memgraph::query::v2

483
src/query/v2/dump.cpp Normal file
View File

@ -0,0 +1,483 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/v2/dump.hpp"
#include <iomanip>
#include <limits>
#include <map>
#include <optional>
#include <ostream>
#include <utility>
#include <vector>
#include <fmt/format.h>
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/db_accessor.hpp"
#include "query/v2/exceptions.hpp"
#include "query/v2/stream.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/storage.hpp"
#include "utils/algorithm.hpp"
#include "utils/logging.hpp"
#include "utils/string.hpp"
#include "utils/temporal.hpp"
namespace memgraph::query::v2 {
namespace {
// Property that is used to make a difference among vertices. It is added to
// property set of vertices to match edges and removed after the entire graph
// is built.
const char *kInternalPropertyId = "__mg_id__";
// Label that is attached to each vertex and is used for easier creation of
// index on internal property id.
const char *kInternalVertexLabel = "__mg_vertex__";
/// A helper function that escapes label, edge type and property names.
std::string EscapeName(const std::string_view value) {
std::string out;
out.reserve(value.size() + 2);
out.append(1, '`');
for (auto c : value) {
if (c == '`') {
out.append("``");
} else {
out.append(1, c);
}
}
out.append(1, '`');
return out;
}
void DumpPreciseDouble(std::ostream *os, double value) {
// A temporary stream is used to keep precision of the original output
// stream unchanged.
std::ostringstream temp_oss;
temp_oss << std::setprecision(std::numeric_limits<double>::max_digits10) << value;
*os << temp_oss.str();
}
namespace {
void DumpDate(std::ostream &os, const storage::v3::TemporalData &value) {
utils::Date date(value.microseconds);
os << "DATE(\"" << date << "\")";
}
void DumpLocalTime(std::ostream &os, const storage::v3::TemporalData &value) {
utils::LocalTime lt(value.microseconds);
os << "LOCALTIME(\"" << lt << "\")";
}
void DumpLocalDateTime(std::ostream &os, const storage::v3::TemporalData &value) {
utils::LocalDateTime ldt(value.microseconds);
os << "LOCALDATETIME(\"" << ldt << "\")";
}
void DumpDuration(std::ostream &os, const storage::v3::TemporalData &value) {
utils::Duration dur(value.microseconds);
os << "DURATION(\"" << dur << "\")";
}
void DumpTemporalData(std::ostream &os, const storage::v3::TemporalData &value) {
switch (value.type) {
case storage::v3::TemporalType::Date: {
DumpDate(os, value);
return;
}
case storage::v3::TemporalType::LocalTime: {
DumpLocalTime(os, value);
return;
}
case storage::v3::TemporalType::LocalDateTime: {
DumpLocalDateTime(os, value);
return;
}
case storage::v3::TemporalType::Duration: {
DumpDuration(os, value);
return;
}
}
}
} // namespace
void DumpPropertyValue(std::ostream *os, const storage::v3::PropertyValue &value) {
switch (value.type()) {
case storage::v3::PropertyValue::Type::Null:
*os << "Null";
return;
case storage::v3::PropertyValue::Type::Bool:
*os << (value.ValueBool() ? "true" : "false");
return;
case storage::v3::PropertyValue::Type::String:
*os << utils::Escape(value.ValueString());
return;
case storage::v3::PropertyValue::Type::Int:
*os << value.ValueInt();
return;
case storage::v3::PropertyValue::Type::Double:
DumpPreciseDouble(os, value.ValueDouble());
return;
case storage::v3::PropertyValue::Type::List: {
*os << "[";
const auto &list = value.ValueList();
utils::PrintIterable(*os, list, ", ", [](auto &os, const auto &item) { DumpPropertyValue(&os, item); });
*os << "]";
return;
}
case storage::v3::PropertyValue::Type::Map: {
*os << "{";
const auto &map = value.ValueMap();
utils::PrintIterable(*os, map, ", ", [](auto &os, const auto &kv) {
os << EscapeName(kv.first) << ": ";
DumpPropertyValue(&os, kv.second);
});
*os << "}";
return;
}
case storage::v3::PropertyValue::Type::TemporalData: {
DumpTemporalData(*os, value.ValueTemporalData());
return;
}
}
}
void DumpProperties(std::ostream *os, query::v2::DbAccessor *dba,
const std::map<storage::v3::PropertyId, storage::v3::PropertyValue> &store,
std::optional<int64_t> property_id = std::nullopt) {
*os << "{";
if (property_id) {
*os << kInternalPropertyId << ": " << *property_id;
if (store.size() > 0) *os << ", ";
}
utils::PrintIterable(*os, store, ", ", [&dba](auto &os, const auto &kv) {
os << EscapeName(dba->PropertyToName(kv.first)) << ": ";
DumpPropertyValue(&os, kv.second);
});
*os << "}";
}
void DumpVertex(std::ostream *os, query::v2::DbAccessor *dba, const query::v2::VertexAccessor &vertex) {
*os << "CREATE (";
*os << ":" << kInternalVertexLabel;
auto maybe_labels = vertex.Labels(storage::v3::View::OLD);
if (maybe_labels.HasError()) {
switch (maybe_labels.GetError()) {
case storage::v3::Error::DELETED_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get labels from a deleted node.");
case storage::v3::Error::NONEXISTENT_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get labels from a node that doesn't exist.");
case storage::v3::Error::SERIALIZATION_ERROR:
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::PROPERTIES_DISABLED:
throw query::v2::QueryRuntimeException("Unexpected error when getting labels.");
}
}
for (const auto &label : *maybe_labels) {
*os << ":" << EscapeName(dba->LabelToName(label));
}
*os << " ";
auto maybe_props = vertex.Properties(storage::v3::View::OLD);
if (maybe_props.HasError()) {
switch (maybe_props.GetError()) {
case storage::v3::Error::DELETED_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get properties from a deleted object.");
case storage::v3::Error::NONEXISTENT_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get properties from a node that doesn't exist.");
case storage::v3::Error::SERIALIZATION_ERROR:
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::PROPERTIES_DISABLED:
throw query::v2::QueryRuntimeException("Unexpected error when getting properties.");
}
}
DumpProperties(os, dba, *maybe_props, vertex.CypherId());
*os << ");";
}
void DumpEdge(std::ostream *os, query::v2::DbAccessor *dba, const query::v2::EdgeAccessor &edge) {
*os << "MATCH ";
*os << "(u:" << kInternalVertexLabel << "), ";
*os << "(v:" << kInternalVertexLabel << ")";
*os << " WHERE ";
*os << "u." << kInternalPropertyId << " = " << edge.From().CypherId();
*os << " AND ";
*os << "v." << kInternalPropertyId << " = " << edge.To().CypherId() << " ";
*os << "CREATE (u)-[";
*os << ":" << EscapeName(dba->EdgeTypeToName(edge.EdgeType()));
auto maybe_props = edge.Properties(storage::v3::View::OLD);
if (maybe_props.HasError()) {
switch (maybe_props.GetError()) {
case storage::v3::Error::DELETED_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get properties from a deleted object.");
case storage::v3::Error::NONEXISTENT_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get properties from an edge that doesn't exist.");
case storage::v3::Error::SERIALIZATION_ERROR:
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::PROPERTIES_DISABLED:
throw query::v2::QueryRuntimeException("Unexpected error when getting properties.");
}
}
if (maybe_props->size() > 0) {
*os << " ";
DumpProperties(os, dba, *maybe_props);
}
*os << "]->(v);";
}
void DumpLabelIndex(std::ostream *os, query::v2::DbAccessor *dba, const storage::v3::LabelId label) {
*os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << ";";
}
void DumpLabelPropertyIndex(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label,
storage::v3::PropertyId property) {
*os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << "(" << EscapeName(dba->PropertyToName(property))
<< ");";
}
void DumpExistenceConstraint(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label,
storage::v3::PropertyId property) {
*os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT EXISTS (u."
<< EscapeName(dba->PropertyToName(property)) << ");";
}
void DumpUniqueConstraint(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label,
const std::set<storage::v3::PropertyId> &properties) {
*os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT ";
utils::PrintIterable(*os, properties, ", ", [&dba](auto &stream, const auto &property) {
stream << "u." << EscapeName(dba->PropertyToName(property));
});
*os << " IS UNIQUE;";
}
} // namespace
PullPlanDump::PullPlanDump(DbAccessor *dba)
: dba_(dba),
vertices_iterable_(dba->Vertices(storage::v3::View::OLD)),
pull_chunks_{// Dump all label indices
CreateLabelIndicesPullChunk(),
// Dump all label property indices
CreateLabelPropertyIndicesPullChunk(),
// Create internal index for faster edge creation
CreateInternalIndexPullChunk(),
// Dump all vertices
CreateVertexPullChunk(),
// Dump all edges
CreateEdgePullChunk(),
// Drop the internal index
CreateDropInternalIndexPullChunk(),
// Internal index cleanup
CreateInternalIndexCleanupPullChunk()} {}
bool PullPlanDump::Pull(AnyStream *stream, std::optional<int> n) {
// Iterate all functions that stream some results.
// Each function should return number of results it streamed after it
// finishes. If the function did not finish streaming all the results,
// std::nullopt should be returned because n results have already been sent.
while (current_chunk_index_ < pull_chunks_.size() && (!n || *n > 0)) {
const auto maybe_streamed_count = pull_chunks_[current_chunk_index_](stream, n);
if (!maybe_streamed_count) {
// n wasn't large enough to stream all the results from the current chunk
break;
}
if (n) {
// chunk finished streaming its results
// subtract number of results streamed in current pull
// so we know how many results we need to stream from future
// chunks.
*n -= *maybe_streamed_count;
}
++current_chunk_index_;
}
return current_chunk_index_ == pull_chunks_.size();
}
PullPlanDump::PullChunk PullPlanDump::CreateLabelIndicesPullChunk() {
// Dump all label indices
return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the construction of indices vectors
if (!indices_info_) {
indices_info_.emplace(dba_->ListAllIndices());
}
const auto &label = indices_info_->label;
size_t local_counter = 0;
while (global_index < label.size() && (!n || local_counter < *n)) {
std::ostringstream os;
DumpLabelIndex(&os, dba_, label[global_index]);
stream->Result({TypedValue(os.str())});
++global_index;
++local_counter;
}
if (global_index == label.size()) {
return local_counter;
}
return std::nullopt;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateLabelPropertyIndicesPullChunk() {
return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the construction of indices vectors
if (!indices_info_) {
indices_info_.emplace(dba_->ListAllIndices());
}
const auto &label_property = indices_info_->label_property;
size_t local_counter = 0;
while (global_index < label_property.size() && (!n || local_counter < *n)) {
std::ostringstream os;
const auto &label_property_index = label_property[global_index];
DumpLabelPropertyIndex(&os, dba_, label_property_index.first, label_property_index.second);
stream->Result({TypedValue(os.str())});
++global_index;
++local_counter;
}
if (global_index == label_property.size()) {
return local_counter;
}
return std::nullopt;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexPullChunk() {
return [this](AnyStream *stream, std::optional<int>) mutable -> std::optional<size_t> {
if (vertices_iterable_.begin() != vertices_iterable_.end()) {
std::ostringstream os;
os << "CREATE INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");";
stream->Result({TypedValue(os.str())});
internal_index_created_ = true;
return 1;
}
return 0;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateVertexPullChunk() {
return [this, maybe_current_iter = std::optional<VertexAccessorIterableIterator>{}](
AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the call of begin() function
// If multiple begins are called before an iteration,
// one iteration will make the rest of iterators be in undefined
// states.
if (!maybe_current_iter) {
maybe_current_iter.emplace(vertices_iterable_.begin());
}
auto &current_iter{*maybe_current_iter};
size_t local_counter = 0;
while (current_iter != vertices_iterable_.end() && (!n || local_counter < *n)) {
std::ostringstream os;
DumpVertex(&os, dba_, *current_iter);
stream->Result({TypedValue(os.str())});
++local_counter;
++current_iter;
}
if (current_iter == vertices_iterable_.end()) {
return local_counter;
}
return std::nullopt;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateEdgePullChunk() {
return [this, maybe_current_vertex_iter = std::optional<VertexAccessorIterableIterator>{},
// we need to save the iterable which contains list of accessor so
// our saved iterator is valid in the next run
maybe_edge_iterable = std::shared_ptr<EdgeAccessorIterable>{nullptr},
maybe_current_edge_iter = std::optional<EdgeAccessorIterableIterator>{}](
AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the call of begin() function
// If multiple begins are called before an iteration,
// one iteration will make the rest of iterators be in undefined
// states.
if (!maybe_current_vertex_iter) {
maybe_current_vertex_iter.emplace(vertices_iterable_.begin());
}
auto &current_vertex_iter{*maybe_current_vertex_iter};
size_t local_counter = 0U;
for (; current_vertex_iter != vertices_iterable_.end() && (!n || local_counter < *n); ++current_vertex_iter) {
const auto &vertex = *current_vertex_iter;
// If we have a saved iterable from a previous pull
// we need to use the same iterable
if (!maybe_edge_iterable) {
maybe_edge_iterable = std::make_shared<EdgeAccessorIterable>(vertex.OutEdges(storage::v3::View::OLD));
}
auto &maybe_edges = *maybe_edge_iterable;
MG_ASSERT(maybe_edges.HasValue(), "Invalid database state!");
auto current_edge_iter = maybe_current_edge_iter ? *maybe_current_edge_iter : maybe_edges->begin();
for (; current_edge_iter != maybe_edges->end() && (!n || local_counter < *n); ++current_edge_iter) {
std::ostringstream os;
DumpEdge(&os, dba_, *current_edge_iter);
stream->Result({TypedValue(os.str())});
++local_counter;
}
if (current_edge_iter != maybe_edges->end()) {
maybe_current_edge_iter.emplace(current_edge_iter);
return std::nullopt;
}
maybe_current_edge_iter = std::nullopt;
maybe_edge_iterable = nullptr;
}
if (current_vertex_iter == vertices_iterable_.end()) {
return local_counter;
}
return std::nullopt;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateDropInternalIndexPullChunk() {
return [this](AnyStream *stream, std::optional<int>) {
if (internal_index_created_) {
std::ostringstream os;
os << "DROP INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");";
stream->Result({TypedValue(os.str())});
return 1;
}
return 0;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexCleanupPullChunk() {
return [this](AnyStream *stream, std::optional<int>) {
if (internal_index_created_) {
std::ostringstream os;
os << "MATCH (u) REMOVE u:" << kInternalVertexLabel << ", u." << kInternalPropertyId << ";";
stream->Result({TypedValue(os.str())});
return 1;
}
return 0;
};
}
void DumpDatabaseToCypherQueries(query::v2::DbAccessor *dba, AnyStream *stream) { PullPlanDump(dba).Pull(stream, {}); }
} // namespace memgraph::query::v2

63
src/query/v2/dump.hpp Normal file
View File

@ -0,0 +1,63 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <ostream>
#include "query/v2/db_accessor.hpp"
#include "query/v2/stream.hpp"
#include "storage/v3/storage.hpp"
namespace memgraph::query::v2 {
void DumpDatabaseToCypherQueries(query::v2::DbAccessor *dba, AnyStream *stream);
struct PullPlanDump {
explicit PullPlanDump(query::v2::DbAccessor *dba);
/// Pull the dump results lazily
/// @return true if all results were returned, false otherwise
bool Pull(AnyStream *stream, std::optional<int> n);
private:
query::v2::DbAccessor *dba_ = nullptr;
std::optional<storage::v3::IndicesInfo> indices_info_ = std::nullopt;
using VertexAccessorIterable = decltype(std::declval<query::v2::DbAccessor>().Vertices(storage::v3::View::OLD));
using VertexAccessorIterableIterator = decltype(std::declval<VertexAccessorIterable>().begin());
using EdgeAccessorIterable = decltype(std::declval<VertexAccessor>().OutEdges(storage::v3::View::OLD));
using EdgeAccessorIterableIterator = decltype(std::declval<EdgeAccessorIterable>().GetValue().begin());
VertexAccessorIterable vertices_iterable_;
bool internal_index_created_ = false;
size_t current_chunk_index_ = 0;
using PullChunk = std::function<std::optional<size_t>(AnyStream *stream, std::optional<int> n)>;
// We define every part of the dump query in a self contained function.
// Each functions is responsible of keeping track of its execution status.
// If a function did finish its execution, it should return number of results
// it streamed so we know how many rows should be pulled from the next
// function, otherwise std::nullopt is returned.
std::vector<PullChunk> pull_chunks_;
PullChunk CreateLabelIndicesPullChunk();
PullChunk CreateLabelPropertyIndicesPullChunk();
PullChunk CreateInternalIndexPullChunk();
PullChunk CreateVertexPullChunk();
PullChunk CreateEdgePullChunk();
PullChunk CreateDropInternalIndexPullChunk();
PullChunk CreateInternalIndexCleanupPullChunk();
};
} // namespace memgraph::query::v2

235
src/query/v2/exceptions.hpp Normal file
View File

@ -0,0 +1,235 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "utils/exceptions.hpp"
#include <fmt/format.h>
namespace memgraph::query::v2 {
/**
* @brief Base class of all query language related exceptions. All exceptions
* derived from this one will be interpreted as ClientError-s, i. e. if client
* executes same query again without making modifications to the database data,
* query will fail again.
*/
class QueryException : public utils::BasicException {
using utils::BasicException::BasicException;
};
class LexingException : public QueryException {
public:
using QueryException::QueryException;
LexingException() : QueryException("") {}
};
class SyntaxException : public QueryException {
public:
using QueryException::QueryException;
SyntaxException() : QueryException("") {}
};
// TODO: Figure out what information to put in exception.
// Error reporting is tricky since we get stripped query and position of error
// in original query is not same as position of error in stripped query. Most
// correct approach would be to do semantic analysis with original query even
// for already hashed queries, but that has obvious performance issues. Other
// approach would be to report some of the semantic errors in runtime of the
// query and only report line numbers of semantic errors (not position in the
// line) if multiple line strings are not allowed by grammar. We could also
// print whole line that contains error instead of specifying line number.
class SemanticException : public QueryException {
public:
using QueryException::QueryException;
SemanticException() : QueryException("") {}
};
class UnboundVariableError : public SemanticException {
public:
explicit UnboundVariableError(const std::string &name) : SemanticException("Unbound variable: " + name + ".") {}
};
class RedeclareVariableError : public SemanticException {
public:
explicit RedeclareVariableError(const std::string &name) : SemanticException("Redeclaring variable: " + name + ".") {}
};
class TypeMismatchError : public SemanticException {
public:
TypeMismatchError(const std::string &name, const std::string &datum, const std::string &expected)
: SemanticException(fmt::format("Type mismatch: {} already defined as {}, expected {}.", name, datum, expected)) {
}
};
class UnprovidedParameterError : public QueryException {
public:
using QueryException::QueryException;
};
class ProfileInMulticommandTxException : public QueryException {
public:
using QueryException::QueryException;
ProfileInMulticommandTxException() : QueryException("PROFILE not allowed in multicommand transactions.") {}
};
class IndexInMulticommandTxException : public QueryException {
public:
using QueryException::QueryException;
IndexInMulticommandTxException() : QueryException("Index manipulation not allowed in multicommand transactions.") {}
};
class ConstraintInMulticommandTxException : public QueryException {
public:
using QueryException::QueryException;
ConstraintInMulticommandTxException()
: QueryException(
"Constraint manipulation not allowed in multicommand "
"transactions.") {}
};
class InfoInMulticommandTxException : public QueryException {
public:
using QueryException::QueryException;
InfoInMulticommandTxException() : QueryException("Info reporting not allowed in multicommand transactions.") {}
};
/**
* An exception for an illegal operation that can not be detected
* before the query starts executing over data.
*/
class QueryRuntimeException : public QueryException {
public:
using QueryException::QueryException;
};
// This one is inherited from BasicException and will be treated as
// TransientError, i. e. client will be encouraged to retry execution because it
// could succeed if executed again.
class HintedAbortError : public utils::BasicException {
public:
using utils::BasicException::BasicException;
HintedAbortError()
: utils::BasicException(
"Transaction was asked to abort, most likely because it was "
"executing longer than time specified by "
"--query-execution-timeout-sec flag.") {}
};
class ExplicitTransactionUsageException : public QueryRuntimeException {
public:
using QueryRuntimeException::QueryRuntimeException;
};
/**
* An exception for serialization error
*/
class TransactionSerializationException : public QueryException {
public:
using QueryException::QueryException;
TransactionSerializationException()
: QueryException(
"Cannot resolve conflicting transactions. You can retry this transaction when the conflicting transaction "
"is finished") {}
};
class ReconstructionException : public QueryException {
public:
ReconstructionException()
: QueryException(
"Record invalid after WITH clause. Most likely deleted by a "
"preceeding DELETE.") {}
};
class RemoveAttachedVertexException : public QueryRuntimeException {
public:
RemoveAttachedVertexException()
: QueryRuntimeException(
"Failed to remove node because of it's existing "
"connections. Consider using DETACH DELETE.") {}
};
class UserModificationInMulticommandTxException : public QueryException {
public:
UserModificationInMulticommandTxException()
: QueryException("Authentication clause not allowed in multicommand transactions.") {}
};
class InvalidArgumentsException : public QueryException {
public:
InvalidArgumentsException(const std::string &argument_name, const std::string &message)
: QueryException(fmt::format("Invalid arguments sent: {} - {}", argument_name, message)) {}
};
class ReplicationModificationInMulticommandTxException : public QueryException {
public:
ReplicationModificationInMulticommandTxException()
: QueryException("Replication clause not allowed in multicommand transactions.") {}
};
class LockPathModificationInMulticommandTxException : public QueryException {
public:
LockPathModificationInMulticommandTxException()
: QueryException("Lock path query not allowed in multicommand transactions.") {}
};
class FreeMemoryModificationInMulticommandTxException : public QueryException {
public:
FreeMemoryModificationInMulticommandTxException()
: QueryException("Free memory query not allowed in multicommand transactions.") {}
};
class TriggerModificationInMulticommandTxException : public QueryException {
public:
TriggerModificationInMulticommandTxException()
: QueryException("Trigger queries not allowed in multicommand transactions.") {}
};
class StreamQueryInMulticommandTxException : public QueryException {
public:
StreamQueryInMulticommandTxException()
: QueryException("Stream queries are not allowed in multicommand transactions.") {}
};
class IsolationLevelModificationInMulticommandTxException : public QueryException {
public:
IsolationLevelModificationInMulticommandTxException()
: QueryException("Isolation level cannot be modified in multicommand transactions.") {}
};
class CreateSnapshotInMulticommandTxException final : public QueryException {
public:
CreateSnapshotInMulticommandTxException()
: QueryException("Snapshot cannot be created in multicommand transactions.") {}
};
class SettingConfigInMulticommandTxException final : public QueryException {
public:
SettingConfigInMulticommandTxException()
: QueryException("Settings cannot be changed or fetched in multicommand transactions.") {}
};
class VersionInfoInMulticommandTxException : public QueryException {
public:
VersionInfoInMulticommandTxException()
: QueryException("Version info query not allowed in multicommand transactions.") {}
};
/**
* An exception for an illegal operation that violates schema
*/
class SchemaViolationException : public QueryRuntimeException {
public:
using QueryRuntimeException::QueryRuntimeException;
};
} // namespace memgraph::query::v2

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,148 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/v2/bindings/ast_visitor.hpp"
#include "query/v2/frontend/ast/ast.hpp"
#include "utils/memory.hpp"
namespace memgraph::query::v2 {
class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVisitor {
public:
using HierarchicalTreeVisitor::PostVisit;
using HierarchicalTreeVisitor::PreVisit;
using HierarchicalTreeVisitor::Visit;
using QueryVisitor<void>::Visit;
std::vector<AuthQuery::Privilege> privileges() { return privileges_; }
void Visit(IndexQuery &) override { AddPrivilege(AuthQuery::Privilege::INDEX); }
void Visit(AuthQuery &) override { AddPrivilege(AuthQuery::Privilege::AUTH); }
void Visit(ExplainQuery &query) override { query.cypher_query_->Accept(*this); }
void Visit(ProfileQuery &query) override { query.cypher_query_->Accept(*this); }
void Visit(InfoQuery &info_query) override {
switch (info_query.info_type_) {
case InfoQuery::InfoType::INDEX:
// TODO: This should be INDEX | STATS, but we don't have support for
// *or* with privileges.
AddPrivilege(AuthQuery::Privilege::INDEX);
break;
case InfoQuery::InfoType::STORAGE:
AddPrivilege(AuthQuery::Privilege::STATS);
break;
case InfoQuery::InfoType::CONSTRAINT:
// TODO: This should be CONSTRAINT | STATS, but we don't have support
// for *or* with privileges.
AddPrivilege(AuthQuery::Privilege::CONSTRAINT);
break;
}
}
void Visit(ConstraintQuery &constraint_query) override { AddPrivilege(AuthQuery::Privilege::CONSTRAINT); }
void Visit(CypherQuery &query) override {
query.single_query_->Accept(*this);
for (auto *cypher_union : query.cypher_unions_) {
cypher_union->Accept(*this);
}
}
void Visit(DumpQuery &dump_query) override { AddPrivilege(AuthQuery::Privilege::DUMP); }
void Visit(LockPathQuery &lock_path_query) override { AddPrivilege(AuthQuery::Privilege::DURABILITY); }
void Visit(FreeMemoryQuery &free_memory_query) override { AddPrivilege(AuthQuery::Privilege::FREE_MEMORY); }
void Visit(TriggerQuery &trigger_query) override { AddPrivilege(AuthQuery::Privilege::TRIGGER); }
void Visit(StreamQuery &stream_query) override { AddPrivilege(AuthQuery::Privilege::STREAM); }
void Visit(ReplicationQuery &replication_query) override { AddPrivilege(AuthQuery::Privilege::REPLICATION); }
void Visit(IsolationLevelQuery &isolation_level_query) override { AddPrivilege(AuthQuery::Privilege::CONFIG); }
void Visit(CreateSnapshotQuery &create_snapshot_query) override { AddPrivilege(AuthQuery::Privilege::DURABILITY); }
void Visit(SettingQuery & /*setting_query*/) override { AddPrivilege(AuthQuery::Privilege::CONFIG); }
void Visit(VersionQuery & /*version_query*/) override { AddPrivilege(AuthQuery::Privilege::STATS); }
void Visit(SchemaQuery & /*schema_query*/) override { AddPrivilege(AuthQuery::Privilege::SCHEMA); }
bool PreVisit(Create & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::CREATE);
return false;
}
bool PreVisit(CallProcedure & /*procedure*/) override {
return false;
}
bool PreVisit(Delete & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::DELETE);
return false;
}
bool PreVisit(Match & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::MATCH);
return false;
}
bool PreVisit(Merge & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::MERGE);
return false;
}
bool PreVisit(SetProperty & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::SET);
return false;
}
bool PreVisit(SetProperties & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::SET);
return false;
}
bool PreVisit(SetLabels & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::SET);
return false;
}
bool PreVisit(RemoveProperty & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::REMOVE);
return false;
}
bool PreVisit(RemoveLabels & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::REMOVE);
return false;
}
bool PreVisit(LoadCsv & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::READ_FILE);
return false;
}
bool Visit(Identifier & /*unused*/) override { return true; }
bool Visit(PrimitiveLiteral & /*unused*/) override { return true; }
bool Visit(ParameterLookup & /*unused*/) override { return true; }
private:
void AddPrivilege(AuthQuery::Privilege privilege) {
if (!utils::Contains(privileges_, privilege)) {
privileges_.push_back(privilege);
}
}
std::vector<AuthQuery::Privilege> privileges_;
};
std::vector<AuthQuery::Privilege> GetRequiredPrivileges(Query *query) {
PrivilegeExtractor extractor;
query->Accept(extractor);
return extractor.privileges();
}
} // namespace memgraph::query::v2

View File

@ -0,0 +1,18 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/frontend/ast/ast.hpp"
namespace memgraph::query::v2 {
std::vector<AuthQuery::Privilege> GetRequiredPrivileges(Query *query);
} // namespace memgraph::query::v2

View File

@ -0,0 +1,535 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/v2/frontend/stripped.hpp"
#include <cctype>
#include <cstdint>
#include <iostream>
#include <span>
#include <string>
#include <vector>
#include "expr/parsing.hpp"
#include "parser/opencypher/generated/MemgraphCypher.h"
#include "parser/opencypher/generated/MemgraphCypherBaseVisitor.h"
#include "parser/opencypher/generated/MemgraphCypherLexer.h"
#include "parser/stripped_lexer_constants.hpp"
#include "query/v2/exceptions.hpp"
#include "utils/fnv.hpp"
#include "utils/logging.hpp"
#include "utils/string.hpp"
namespace memgraph::query::v2::frontend {
using namespace parser::lexer_constants; // NOLINT(google-build-using-namespace)
StrippedQuery::StrippedQuery(const std::string &query) : original_(query) {
enum class Token {
UNMATCHED,
KEYWORD, // Including true, false and null.
SPECIAL, // +, .., +=, (, { and so on.
STRING,
INT, // Decimal, octal and hexadecimal.
REAL,
PARAMETER,
ESCAPED_NAME,
UNESCAPED_NAME,
SPACE
};
std::vector<std::pair<Token, std::string>> tokens;
std::string unstripped_chunk;
for (int i = 0; i < static_cast<int>(original_.size());) {
Token token = Token::UNMATCHED;
int len = 0;
auto update = [&](int new_len, Token new_token) {
if (new_len > len) {
len = new_len;
token = new_token;
}
};
update(MatchKeyword(i), Token::KEYWORD);
update(MatchSpecial(i), Token::SPECIAL);
update(MatchString(i), Token::STRING);
update(MatchDecimalInt(i), Token::INT);
update(MatchOctalInt(i), Token::INT);
update(MatchHexadecimalInt(i), Token::INT);
update(MatchReal(i), Token::REAL);
update(MatchParameter(i), Token::PARAMETER);
update(MatchEscapedName(i), Token::ESCAPED_NAME);
update(MatchUnescapedName(i), Token::UNESCAPED_NAME);
update(MatchWhitespaceAndComments(i), Token::SPACE);
if (token == Token::UNMATCHED) throw LexingException("Invalid query.");
tokens.emplace_back(token, original_.substr(i, len));
i += len;
// If we notice execute, we possibly create a trigger which has defined statements.
// The statements will be parsed separately later on so we skip it for now.
if (utils::IEquals(tokens.back().second, "execute")) {
// check if it's CREATE TRIGGER query
std::span token_span{tokens};
// query could start with spaces and/or comments
if (token_span.front().first == Token::SPACE) {
token_span = token_span.subspan(1);
}
// we need to check that first and third elements are correct keywords
// CREATE<SPACE>TRIGGER<SPACE>trigger-name...EXECUTE
// trigger-name (5th element) can also be "execute" so we verify that the size is larger than 5
if (token_span.size() > 5 && utils::IEquals(token_span[0].second, "create") &&
utils::IEquals(token_span[2].second, "trigger")) {
unstripped_chunk = original_.substr(i);
break;
}
}
}
std::vector<std::string> token_strings;
// A helper function that stores literal and its token position in a
// literals_. In stripped query text literal is replaced with a new_value.
// new_value can be any value that is lexed as a literal.
auto replace_stripped = [this, &token_strings](int position, const auto &value, const std::string &new_value) {
literals_.Add(position, storage::v3::PropertyValue(value));
token_strings.push_back(new_value);
};
// Copy original tokens because we need to use original case in named
// expressions and keywords in tokens will be lowercased in the next loop.
auto original_tokens = tokens;
// For every token in original query remember token index in stripped query.
std::vector<int> position_mapping(tokens.size(), -1);
// Convert tokens to strings, perform filtering, store literals and nonaliased
// named expressions in return.
for (int i = 0; i < static_cast<int>(tokens.size()); ++i) {
auto &token = tokens[i];
// We need to shift token index for every parameter since antlr's parser
// thinks of parameter as two tokens.
int token_index = token_strings.size() + parameters_.size();
switch (token.first) {
case Token::UNMATCHED:
LOG_FATAL("Shouldn't happen");
case Token::KEYWORD: {
// We don't strip NULL, since it can appear in special expressions
// like IS NULL and IS NOT NULL, but we strip true and false keywords.
if (utils::IEquals(token.second, "true")) {
replace_stripped(token_index, true, kStrippedBooleanToken);
} else if (utils::IEquals(token.second, "false")) {
replace_stripped(token_index, false, kStrippedBooleanToken);
} else {
token_strings.push_back(token.second);
}
} break;
case Token::SPACE:
break;
case Token::STRING:
replace_stripped(token_index, expr::ParseStringLiteral(token.second), kStrippedStringToken);
break;
case Token::INT:
replace_stripped(token_index, expr::ParseIntegerLiteral(token.second), kStrippedIntToken);
break;
case Token::REAL:
replace_stripped(token_index, expr::ParseDoubleLiteral(token.second), kStrippedDoubleToken);
break;
case Token::SPECIAL:
case Token::ESCAPED_NAME:
case Token::UNESCAPED_NAME:
token_strings.push_back(token.second);
break;
case Token::PARAMETER:
parameters_[token_index] = expr::ParseParameter(token.second);
token_strings.push_back(token.second);
break;
}
if (token.first != Token::SPACE) {
position_mapping[i] = token_index;
}
}
if (!unstripped_chunk.empty()) {
token_strings.push_back(std::move(unstripped_chunk));
}
query_ = utils::Join(token_strings, " ");
hash_ = utils::Fnv(query_);
auto it = tokens.begin();
while (it != tokens.end()) {
// Store nonaliased named expressions in returns in named_exprs_.
it = std::find_if(it, tokens.end(),
[](const std::pair<Token, std::string> &a) { return utils::IEquals(a.second, "return"); });
// There is no RETURN so there is nothing to do here.
if (it == tokens.end()) return;
// Skip RETURN;
++it;
// Now we need to parse cypherReturn production from opencypher grammar.
// Skip leading whitespaces and DISTINCT statemant if there is one.
while (it != tokens.end() && it->first == Token::SPACE) {
++it;
}
if (it != tokens.end() && utils::IEquals(it->second, "distinct")) {
++it;
}
// If the query is invalid, either antlr parser or cypher_main_visitor will
// report an error.
// TODO: we shouldn't rely on the fact that those checks will be done
// after this step. We should do them here.
while (it < tokens.end()) {
// Disregard leading whitespace
while (it != tokens.end() && it->first == Token::SPACE) {
++it;
}
// There is only whitespace, nothing to do...
if (it == tokens.end()) break;
bool has_as = false;
auto last_non_space = it;
auto jt = it;
// We should track number of opened braces and parantheses so that we can
// recognize if comma is a named expression separator or part of the
// list literal / function call.
int num_open_braces = 0;
int num_open_parantheses = 0;
int num_open_brackets = 0;
for (;
jt != tokens.end() && (jt->second != "," || num_open_braces || num_open_parantheses || num_open_brackets) &&
!utils::IEquals(jt->second, "order") && !utils::IEquals(jt->second, "skip") &&
!utils::IEquals(jt->second, "limit") && !utils::IEquals(jt->second, "union") &&
!utils::IEquals(jt->second, "query") && jt->second != ";";
++jt) {
if (jt->second == "(") {
++num_open_parantheses;
} else if (jt->second == ")") {
--num_open_parantheses;
} else if (jt->second == "[") {
++num_open_braces;
} else if (jt->second == "]") {
--num_open_braces;
} else if (jt->second == "{") {
++num_open_brackets;
} else if (jt->second == "}") {
--num_open_brackets;
}
has_as |= utils::IEquals(jt->second, "as");
if (jt->first != Token::SPACE) {
last_non_space = jt;
}
}
if (!has_as) {
// Named expression is not aliased. Save string disregarding leading and
// trailing whitespaces.
std::string s;
auto begin_token = it - tokens.begin() + original_tokens.begin();
auto end_token = last_non_space - tokens.begin() + original_tokens.begin() + 1;
for (auto kt = begin_token; kt != end_token; ++kt) {
s += kt->second;
}
named_exprs_[position_mapping[it - tokens.begin()]] = s;
}
if (jt != tokens.end() && jt->second == ",") {
// There are more named expressions.
it = jt + 1;
} else {
// We're done with this return statement
break;
}
}
}
}
std::string GetFirstUtf8Symbol(const char *_s) {
// According to
// https://stackoverflow.com/questions/16260033/reinterpret-cast-between-char-and-stduint8-t-safe
// this checks if casting from const char * to uint8_t is undefined behaviour.
static_assert(std::is_same<std::uint8_t, unsigned char>::value,
"This library requires std::uint8_t to be implemented as "
"unsigned char.");
const uint8_t *s = reinterpret_cast<const uint8_t *>(_s);
if ((*s >> 7) == 0x00) return std::string(_s, _s + 1);
if ((*s >> 5) == 0x06) {
auto *s1 = s + 1;
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
return std::string(_s, _s + 2);
}
if ((*s >> 4) == 0x0e) {
auto *s1 = s + 1;
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
auto *s2 = s + 2;
if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character.");
return std::string(_s, _s + 3);
}
if ((*s >> 3) == 0x1e) {
auto *s1 = s + 1;
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
auto *s2 = s + 2;
if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character.");
auto *s3 = s + 3;
if ((*s3 >> 6) != 0x02) throw LexingException("Invalid character.");
return std::string(_s, _s + 4);
}
throw LexingException("Invalid character.");
}
// Return codepoint of first utf8 symbol and its encoded length.
std::pair<int, int> GetFirstUtf8SymbolCodepoint(const char *_s) {
static_assert(std::is_same<std::uint8_t, unsigned char>::value,
"This library requires std::uint8_t to be implemented as "
"unsigned char.");
const uint8_t *s = reinterpret_cast<const uint8_t *>(_s);
if ((*s >> 7) == 0x00) return {*s & 0x7f, 1};
if ((*s >> 5) == 0x06) {
auto *s1 = s + 1;
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
return {((*s & 0x1f) << 6) | (*s1 & 0x3f), 2};
}
if ((*s >> 4) == 0x0e) {
auto *s1 = s + 1;
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
auto *s2 = s + 2;
if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character.");
return {((*s & 0x0f) << 12) | ((*s1 & 0x3f) << 6) | (*s2 & 0x3f), 3};
}
if ((*s >> 3) == 0x1e) {
auto *s1 = s + 1;
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
auto *s2 = s + 2;
if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character.");
auto *s3 = s + 3;
if ((*s3 >> 6) != 0x02) throw LexingException("Invalid character.");
return {((*s & 0x07) << 18) | ((*s1 & 0x3f) << 12) | ((*s2 & 0x3f) << 6) | (*s3 & 0x3f), 4};
}
throw LexingException("Invalid character.");
}
// From here until end of file there are functions that calculate matches for
// every possible token. Functions are more or less compatible with Cypher.g4
// grammar. Unfortunately, they contain a lof of special cases and shouldn't
// be changed without good reasons.
//
// Here be dragons, do not touch!
// ____ __
// { --.\ | .)%%%)%%
// '-._\\ | (\___ %)%%(%%(%%%
// `\\|{/ ^ _)-%(%%%%)%%;%%%
// .'^^^^^^^ /` %%)%%%%)%%%'
// //\ ) , / '%%%%(%%'
// , _.'/ `\<-- \<
// `^^^` ^^ ^^
int StrippedQuery::MatchKeyword(int start) const { return kKeywords.Match<tolower>(original_.c_str() + start); }
int StrippedQuery::MatchSpecial(int start) const { return kSpecialTokens.Match(original_.c_str() + start); }
int StrippedQuery::MatchString(int start) const {
if (original_[start] != '"' && original_[start] != '\'') return 0;
char start_char = original_[start];
for (auto *p = original_.data() + start + 1; *p; ++p) {
if (*p == start_char) return p - (original_.data() + start) + 1;
if (*p == '\\') {
++p;
if (*p == '\\' || *p == '\'' || *p == '"' || *p == 'B' || *p == 'b' || *p == 'F' || *p == 'f' || *p == 'N' ||
*p == 'n' || *p == 'R' || *p == 'r' || *p == 'T' || *p == 't') {
// Allowed escaped characters.
continue;
} else if (*p == 'U' || *p == 'u') {
int cnt = 0;
auto *r = p + 1;
while (isxdigit(*r) && cnt < 8) {
++cnt;
++r;
}
if (!*r) return 0;
if (cnt < 4) return 0;
if (cnt >= 4 && cnt < 8) {
p += 4;
}
if (cnt >= 8) {
p += 8;
}
} else {
return 0;
}
}
}
return 0;
}
int StrippedQuery::MatchDecimalInt(int start) const {
if (original_[start] == '0') return 1;
int i = start;
while (i < static_cast<int>(original_.size()) && isdigit(original_[i])) {
++i;
}
return i - start;
}
int StrippedQuery::MatchOctalInt(int start) const {
if (original_[start] != '0') return 0;
int i = start + 1;
while (i < static_cast<int>(original_.size()) && '0' <= original_[i] && original_[i] <= '7') {
++i;
}
if (i == start + 1) return 0;
return i - start;
}
int StrippedQuery::MatchHexadecimalInt(int start) const {
if (original_[start] != '0') return 0;
if (start + 1 >= static_cast<int>(original_.size())) return 0;
if (original_[start + 1] != 'x') return 0;
int i = start + 2;
while (i < static_cast<int>(original_.size()) && isxdigit(original_[i])) {
++i;
}
if (i == start + 2) return 0;
return i - start;
}
int StrippedQuery::MatchReal(int start) const {
enum class State { START, BEFORE_DOT, DOT, AFTER_DOT, E, E_MINUS, AFTER_E };
State state = State::START;
auto i = start;
while (i < static_cast<int>(original_.size())) {
if (original_[i] == '.') {
if (state != State::BEFORE_DOT && state != State::START) break;
state = State::DOT;
} else if ('0' <= original_[i] && original_[i] <= '9') {
if (state == State::START) {
state = State::BEFORE_DOT;
} else if (state == State::DOT) {
state = State::AFTER_DOT;
} else if (state == State::E || state == State::E_MINUS) {
state = State::AFTER_E;
}
} else if (original_[i] == 'e' || original_[i] == 'E') {
if (state != State::BEFORE_DOT && state != State::AFTER_DOT) break;
state = State::E;
} else if (original_[i] == '-') {
if (state != State::E) break;
state = State::E_MINUS;
} else {
break;
}
++i;
}
if (state == State::DOT) --i;
if (state == State::E) --i;
if (state == State::E_MINUS) i -= 2;
return i - start;
}
int StrippedQuery::MatchParameter(int start) const {
int len = original_.size();
if (start + 1 == len) return 0;
if (original_[start] != '$') return 0;
int max_len = 0;
max_len = std::max(max_len, MatchUnescapedName(start + 1));
max_len = std::max(max_len, MatchEscapedName(start + 1));
max_len = std::max(max_len, MatchKeyword(start + 1));
max_len = std::max(max_len, MatchDecimalInt(start + 1));
if (max_len == 0) return 0;
return 1 + max_len;
}
int StrippedQuery::MatchEscapedName(int start) const {
int len = original_.size();
int i = start;
while (i < len) {
if (original_[i] != '`') break;
int j = i + 1;
while (j < len && original_[j] != '`') {
++j;
}
if (j == len) break;
i = j + 1;
}
return i - start;
}
int StrippedQuery::MatchUnescapedName(int start) const {
auto i = start;
auto got = GetFirstUtf8SymbolCodepoint(original_.data() + i);
if (got.first >= parser::lexer_constants::kBitsetSize || !kUnescapedNameAllowedStarts[got.first]) {
return 0;
}
i += got.second;
while (i < static_cast<int>(original_.size())) {
got = GetFirstUtf8SymbolCodepoint(original_.data() + i);
if (got.first >= parser::lexer_constants::kBitsetSize || !kUnescapedNameAllowedParts[got.first]) {
break;
}
i += got.second;
}
return i - start;
}
int StrippedQuery::MatchWhitespaceAndComments(int start) const {
enum class State { OUT, IN_LINE_COMMENT, IN_BLOCK_COMMENT };
State state = State::OUT;
int i = start;
int len = original_.size();
// We need to remember at which position comment started because if we fail
// to match comment finish we have a match until comment start position.
int comment_position = -1;
while (i < len) {
if (state == State::OUT) {
auto got = GetFirstUtf8SymbolCodepoint(original_.data() + i);
if (got.first < parser::lexer_constants::kBitsetSize && kSpaceParts[got.first]) {
i += got.second;
} else if (i + 1 < len && original_[i] == '/' && original_[i + 1] == '*') {
comment_position = i;
state = State::IN_BLOCK_COMMENT;
i += 2;
} else if (i + 1 < len && original_[i] == '/' && original_[i + 1] == '/') {
comment_position = i;
if (i + 2 < len) {
// Special case for an empty line comment starting right at the end of
// the query.
state = State::IN_LINE_COMMENT;
}
i += 2;
} else {
break;
}
} else if (state == State::IN_LINE_COMMENT) {
if (original_[i] == '\n') {
state = State::OUT;
++i;
} else if (i + 1 < len && original_[i] == '\r' && original_[i + 1] == '\n') {
state = State::OUT;
i += 2;
} else if (original_[i] == '\r') {
break;
} else if (i + 1 == len) {
state = State::OUT;
++i;
} else {
++i;
}
} else if (state == State::IN_BLOCK_COMMENT) {
if (i + 1 < len && original_[i] == '*' && original_[i + 1] == '/') {
i += 2;
state = State::OUT;
} else {
++i;
}
}
}
if (state != State::OUT) return comment_position - start;
return i - start;
}
} // namespace memgraph::query::v2::frontend

Some files were not shown because too many files have changed in this diff Show More