tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add REDUCE function to openCypher

Browse Source 
Reviewers: florijan, msantl

Reviewed By: florijan

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1175
This commit is contained in:
Teon Banek 2018-02-07 10:57:38 +01:00
parent d05d73f13a
commit fa55c130ba
18 changed files with 265 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -3,6 +3,7 @@
## Next Release
* Snapshot format changed (not backward compatible).
* `reduce` function added.
## v0.9.0

1
docs/user_technical/open-cypher.md
View File

@ -550,6 +550,7 @@ functions.
`endsWith` | Check if the first argument ends with the second.
`contains` | Check if the first argument has an element which is equal to the second argument.
`all` | Check if all elements of a list satisfy a predicate.<br/>The syntax is: `all(variable IN list WHERE predicate)`.<br/> NOTE: Whenever possible, use Memgraph's lambda functions when [matching](#filtering-variable-length-paths) instead.
`reduce` | Accumulate list elements into a single result by applying an expression. The syntax is:<br/>`reduce(accumulator = initial_value, variable IN list | expression)`.
`assert` | Raises an exception reported to the client if the given argument is not `true`.
`counter` | Generates integers that are guaranteed to be unique on the database level, for the given counter name.
`counterSet` | Sets the counter with the given name to the given value.

1
src/query/frontend/ast/ast.cpp
View File

@ -76,6 +76,7 @@ BOOST_CLASS_EXPORT_IMPLEMENT(query::PropertyLookup);
BOOST_CLASS_EXPORT_IMPLEMENT(query::LabelsTest);
BOOST_CLASS_EXPORT_IMPLEMENT(query::Aggregation);
BOOST_CLASS_EXPORT_IMPLEMENT(query::Function);
BOOST_CLASS_EXPORT_IMPLEMENT(query::Reduce);
BOOST_CLASS_EXPORT_IMPLEMENT(query::All);
BOOST_CLASS_EXPORT_IMPLEMENT(query::ParameterLookup);
BOOST_CLASS_EXPORT_IMPLEMENT(query::Create);

78
src/query/frontend/ast/ast.hpp
View File

@ -1391,6 +1391,81 @@ class Aggregation : public BinaryOperator {
const unsigned int);
};
class Reduce : public Expression {
friend class AstTreeStorage;
public:
DEFVISITABLE(TreeVisitor<TypedValue>);
bool Accept(HierarchicalTreeVisitor &visitor) override {
if (visitor.PreVisit(*this)) {
accumulator_->Accept(visitor) && initializer_->Accept(visitor) &&
identifier_->Accept(visitor) && list_->Accept(visitor) &&
expression_->Accept(visitor);
}
return visitor.PostVisit(*this);
}
Reduce *Clone(AstTreeStorage &storage) const override {
return storage.Create<Reduce>(
accumulator_->Clone(storage), initializer_->Clone(storage),
identifier_->Clone(storage), list_->Clone(storage),
expression_->Clone(storage));
}
// None of these should be nullptr after construction.
/// Identifier for the accumulating variable
Identifier *accumulator_ = nullptr;
/// Expression which produces the initial accumulator value.
Expression *initializer_ = nullptr;
/// Identifier for the list element.
Identifier *identifier_ = nullptr;
/// Expression which produces a list which will be reduced.
Expression *list_ = nullptr;
/// Expression which does the reduction, i.e. produces the new accumulator
/// value.
Expression *expression_ = nullptr;
protected:
Reduce(int uid, Identifier *accumulator, Expression *initializer,
Identifier *identifier, Expression *list, Expression *expression)
: Expression(uid),
accumulator_(accumulator),
initializer_(initializer),
identifier_(identifier),
list_(list),
expression_(expression) {}
private:
friend class boost::serialization::access;
BOOST_SERIALIZATION_SPLIT_MEMBER();
template <class TArchive>
void save(TArchive &ar, const unsigned int) const {
ar << boost::serialization::base_object<Expression>(*this);
SavePointer(ar, accumulator_);
SavePointer(ar, initializer_);
SavePointer(ar, identifier_);
SavePointer(ar, list_);
SavePointer(ar, expression_);
}
template <class TArchive>
void load(TArchive &ar, const unsigned int) {
ar >> boost::serialization::base_object<Expression>(*this);
LoadPointer(ar, accumulator_);
LoadPointer(ar, initializer_);
LoadPointer(ar, identifier_);
LoadPointer(ar, list_);
LoadPointer(ar, expression_);
}
template <class TArchive>
friend void boost::serialization::load_construct_data(TArchive &, Reduce *,
const unsigned int);
};
// TODO: Think about representing All and Any as Reduce.
class All : public Expression {
friend class AstTreeStorage;
@ -2847,6 +2922,8 @@ LOAD_AND_CONSTRUCT(query::LabelsTest, 0, nullptr,
LOAD_AND_CONSTRUCT(query::Function, 0);
LOAD_AND_CONSTRUCT(query::Aggregation, 0, nullptr, nullptr,
query::Aggregation::Op::COUNT);
LOAD_AND_CONSTRUCT(query::Reduce, 0, nullptr, nullptr, nullptr, nullptr,
nullptr);
LOAD_AND_CONSTRUCT(query::All, 0, nullptr, nullptr, nullptr);
LOAD_AND_CONSTRUCT(query::ParameterLookup, 0);
LOAD_AND_CONSTRUCT(query::NamedExpression, 0);
@ -2906,6 +2983,7 @@ BOOST_CLASS_EXPORT_KEY(query::PropertyLookup);
BOOST_CLASS_EXPORT_KEY(query::LabelsTest);
BOOST_CLASS_EXPORT_KEY(query::Aggregation);
BOOST_CLASS_EXPORT_KEY(query::Function);
BOOST_CLASS_EXPORT_KEY(query::Reduce);
BOOST_CLASS_EXPORT_KEY(query::All);
BOOST_CLASS_EXPORT_KEY(query::ParameterLookup);
BOOST_CLASS_EXPORT_KEY(query::Create);

13
src/query/frontend/ast/ast_visitor.hpp
View File

@ -14,6 +14,7 @@ class PropertyLookup;
class LabelsTest;
class Aggregation;
class Function;
class Reduce;
class All;
class ParameterLookup;
class Create;
@ -67,8 +68,8 @@ using TreeCompositeVisitor = ::utils::CompositeVisitor<
GreaterEqualOperator, InListOperator, ListMapIndexingOperator,
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator,
IsNullOperator, ListLiteral, MapLiteral, PropertyLookup, LabelsTest,
Aggregation, Function, All, Create, Match, Return, With, Pattern, NodeAtom,
EdgeAtom, Delete, Where, SetProperty, SetProperties, SetLabels,
Aggregation, Function, Reduce, All, Create, Match, Return, With, Pattern,
NodeAtom, EdgeAtom, Delete, Where, SetProperty, SetProperties, SetLabels,
RemoveProperty, RemoveLabels, Merge, Unwind>;
using TreeLeafVisitor = ::utils::LeafVisitor<Identifier, PrimitiveLiteral,
@ -92,9 +93,9 @@ using TreeVisitor = ::utils::Visitor<
LessEqualOperator, GreaterEqualOperator, InListOperator,
ListMapIndexingOperator, ListSlicingOperator, IfOperator, UnaryPlusOperator,
UnaryMinusOperator, IsNullOperator, ListLiteral, MapLiteral, PropertyLookup,
LabelsTest, Aggregation, Function, All, ParameterLookup, Create, Match,
Return, With, Pattern, NodeAtom, EdgeAtom, Delete, Where, SetProperty,
SetProperties, SetLabels, RemoveProperty, RemoveLabels, Merge, Unwind,
Identifier, PrimitiveLiteral, CreateIndex>;
LabelsTest, Aggregation, Function, Reduce, All, ParameterLookup, Create,
Match, Return, With, Pattern, NodeAtom, EdgeAtom, Delete, Where,
SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels, Merge,
Unwind, Identifier, PrimitiveLiteral, CreateIndex>;
} // namespace query

15
src/query/frontend/ast/cypher_main_visitor.cpp
View File

@ -865,6 +865,21 @@ antlrcpp::Any CypherMainVisitor::visitAtom(CypherParser::AtomContext *ctx) {
Where *where = ctx->filterExpression()->where()->accept(this);
return static_cast<Expression *>(
storage_.Create<All>(ident, list_expr, where));
} else if (ctx->REDUCE()) {
auto *accumulator = storage_.Create<Identifier>(
ctx->reduceExpression()->accumulator->accept(this).as<std::string>());
Expression *initializer = ctx->reduceExpression()->initial->accept(this);
auto *ident = storage_.Create<Identifier>(ctx->reduceExpression()
->idInColl()
->variable()
->accept(this)
.as<std::string>());
Expression *list =
ctx->reduceExpression()->idInColl()->expression()->accept(this);
Expression *expr =
ctx->reduceExpression()->expression().back()->accept(this);
return static_cast<Expression *>(
storage_.Create<Reduce>(accumulator, initializer, ident, list, expr));
} else if (ctx->caseExpression()) {
return static_cast<Expression *>(ctx->caseExpression()->accept(this));
}

6
src/query/frontend/opencypher/grammar/Cypher.g4
View File

@ -187,6 +187,7 @@ atom : literal
| patternComprehension
| ( FILTER SP? '(' SP? filterExpression SP? ')' )
| ( EXTRACT SP? '(' SP? filterExpression SP? ( SP? '|' expression )? ')' )
| ( REDUCE SP? '(' SP? reduceExpression SP? ')' )
| ( ALL SP? '(' SP? filterExpression SP? ')' )
| ( ANY SP? '(' SP? filterExpression SP? ')' )
| ( NONE SP? '(' SP? filterExpression SP? ')' )
@ -226,6 +227,8 @@ relationshipsPattern : nodePattern ( SP? patternElementChain )+ ;
filterExpression : idInColl ( SP? where )? ;
reduceExpression : accumulator=variable SP? '=' SP? initial=expression SP? ',' SP? idInColl SP? '|' SP? expression ;
idInColl : variable SP IN SP expression ;
functionInvocation : functionName SP? '(' SP? ( DISTINCT SP? )? ( expression SP? ( ',' SP? expression SP? )* )? ')' ;
@ -327,6 +330,7 @@ symbolicName : UnescapedSymbolicName
| EscapedSymbolicName
| UNION
| ALL
| REDUCE
| OPTIONAL
| MATCH
| UNWIND
@ -380,6 +384,8 @@ UNION : ( 'U' | 'u' ) ( 'N' | 'n' ) ( 'I' | 'i' ) ( 'O' | 'o' ) ( 'N' | 'n' ) ;
ALL : ( 'A' | 'a' ) ( 'L' | 'l' ) ( 'L' | 'l' ) ;
REDUCE : ( 'R' | 'r' ) ( 'E' | 'e' ) ( 'D' | 'd' ) ( 'U' | 'u' ) ( 'C' | 'c' ) ( 'E' | 'e' ) ;
OPTIONAL : ( 'O' | 'o' ) ( 'P' | 'p' ) ( 'T' | 't' ) ( 'I' | 'i' ) ( 'O' | 'o' ) ( 'N' | 'n' ) ( 'A' | 'a' ) ( 'L' | 'l' ) ;
MATCH : ( 'M' | 'm' ) ( 'A' | 'a' ) ( 'T' | 't' ) ( 'C' | 'c' ) ( 'H' | 'h' ) ;

9
src/query/frontend/semantic/symbol_generator.cpp
View File

@ -333,6 +333,15 @@ bool SymbolGenerator::PreVisit(All &all) {
return false;
}
bool SymbolGenerator::PreVisit(Reduce &reduce) {
reduce.initializer_->Accept(*this);
reduce.list_->Accept(*this);
VisitWithIdentifiers(*reduce.expression_,
{reduce.accumulator_, reduce.identifier_});
return false;
}
// Pattern and its subparts.
bool SymbolGenerator::PreVisit(Pattern &pattern) {

1
src/query/frontend/semantic/symbol_generator.hpp
View File

@ -57,6 +57,7 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
bool PreVisit(IfOperator &) override;
bool PostVisit(IfOperator &) override;
bool PreVisit(All &) override;
bool PreVisit(Reduce &) override;
// Pattern and its subparts.
bool PreVisit(Pattern &) override;

3
src/query/frontend/stripped_lexer_constants.hpp
View File

@ -86,7 +86,8 @@ const trie::Trie kKeywords = {
"where", "or", "xor", "and", "not", "in",
"starts", "ends", "contains", "is", "null", "case",
"when", "then", "else", "end", "count", "filter",
"extract", "any", "none", "single", "true", "false"};
"extract", "any", "none", "single", "true", "false",
"reduce"};
// Unicode codepoints that are allowed at the start of the unescaped name.
const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(std::string(

21
src/query/interpret/eval.hpp
View File

@ -354,6 +354,27 @@ class ExpressionEvaluator : public TreeVisitor<TypedValue> {
return function.function()(arguments, db_accessor_);
}
TypedValue Visit(Reduce &reduce) override {
auto list_value = reduce.list_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue::Null;
}
if (list_value.type() != TypedValue::Type::List) {
throw QueryRuntimeException("'REDUCE' expected a list, but got {}",
list_value.type());
}
const auto &list = list_value.Value<std::vector<TypedValue>>();
const auto &element_symbol = symbol_table_.at(*reduce.identifier_);
const auto &accumulator_symbol = symbol_table_.at(*reduce.accumulator_);
auto accumulator = reduce.initializer_->Accept(*this);
for (const auto &element : list) {
frame_[accumulator_symbol] = accumulator;
frame_[element_symbol] = element;
accumulator = reduce.expression_->Accept(*this);
}
return accumulator;
}
TypedValue Visit(All &all) override {
auto list_value = all.list_expression_->Accept(*this);
if (list_value.IsNull()) {

9
src/query/plan/preprocess.hpp
View File

@ -30,6 +30,15 @@ class UsedSymbolsCollector : public HierarchicalTreeVisitor {
return true;
}
bool PostVisit(Reduce &reduce) override {
// Remove the symbols bound by reduce, because we are only interested
// in free (unbound) symbols.
symbols_.erase(symbol_table_.at(*reduce.accumulator_));
symbols_.erase(symbol_table_.at(*reduce.identifier_));
return true;
}
bool Visit(Identifier &ident) override {
symbols_.insert(symbol_table_.at(ident));
return true;

17
src/query/plan/rule_based_planner.cpp
View File

@ -210,6 +210,23 @@ class ReturnBodyContext : public HierarchicalTreeVisitor {
return true;
}
bool PostVisit(Reduce &reduce) override {
// Remove the symbols bound by reduce, because we are only interested
// in free (unbound) symbols.
used_symbols_.erase(symbol_table_.at(*reduce.accumulator_));
used_symbols_.erase(symbol_table_.at(*reduce.identifier_));
DCHECK(has_aggregation_.size() >= 5U)
<< "Expected 5 has_aggregation_ flags for REDUCE arguments";
bool has_aggr = false;
for (int i = 0; i < 5; ++i) {
has_aggr = has_aggr || has_aggregation_.back();
has_aggregation_.pop_back();
}
has_aggregation_.emplace_back(has_aggr);
return true;
}
bool Visit(Identifier &ident) override {
const auto &symbol = symbol_table_.at(ident);
if (!utils::Contains(output_symbols_, symbol)) {

24
tests/qa/tck_engine/tests/memgraph_V1/features/functions.feature
View File

@ -678,6 +678,30 @@ Feature: Functions
"""
Then an error should be raised
Scenario: Reduce test 01:
When executing query:
"""
RETURN reduce(a = true, x IN [1, 2, '3'] | a AND x < 2) AS a
"""
Then the result should be:
| a |
| false |
Scenario: Reduce test 02:
When executing query:
"""
RETURN reduce(s = 0, x IN [1, 2, 3] | s + x) AS s
"""
Then the result should be:
| s |
| 6 |
Scenario: Reduce test 03:
When executing query:
"""
RETURN reduce(a = true, x IN [true, true, '3'] | a AND x) AS a
"""
Then an error should be raised
Scenario: Assert test fail, no message:
Given an empty graph

21
tests/unit/cypher_main_visitor.cpp
View File

@ -1563,6 +1563,27 @@ TYPED_TEST(CypherMainVisitorTest, ReturnAll) {
EXPECT_TRUE(eq);
}
TYPED_TEST(CypherMainVisitorTest, ReturnReduce) {
TypeParam ast_generator("RETURN reduce(sum = 0, x IN [1,2,3] | sum + x)");
auto *query = ast_generator.query_;
ASSERT_TRUE(query->single_query_);
auto *single_query = query->single_query_;
ASSERT_EQ(single_query->clauses_.size(), 1U);
auto *ret = dynamic_cast<Return *>(single_query->clauses_[0]);
ASSERT_TRUE(ret);
ASSERT_EQ(ret->body_.named_expressions.size(), 1U);
auto *reduce =
dynamic_cast<Reduce *>(ret->body_.named_expressions[0]->expression_);
ASSERT_TRUE(reduce);
EXPECT_EQ(reduce->accumulator_->name_, "sum");
CheckLiteral(ast_generator.context_, reduce->initializer_, 0);
EXPECT_EQ(reduce->identifier_->name_, "x");
auto *list_literal = dynamic_cast<ListLiteral *>(reduce->list_);
EXPECT_TRUE(list_literal);
auto *add = dynamic_cast<AdditionOperator *>(reduce->expression_);
EXPECT_TRUE(add);
}
TYPED_TEST(CypherMainVisitorTest, MatchBfsReturn) {
TypeParam ast_generator(
"MATCH (n) -[r:type1|type2 *bfs..10 (e, n|e.prop = 42)]-> (m) RETURN r");

4
tests/unit/query_common.hpp
View File

@ -597,3 +597,7 @@ auto GetMerge(AstTreeStorage &storage, Pattern *pattern, OnMatch on_match,
#define ALL(variable, list, where) \
storage.Create<query::All>(storage.Create<query::Identifier>(variable), \
list, where)
#define REDUCE(accumulator, initializer, variable, list, expr) \
storage.Create<query::Reduce>( \
storage.Create<query::Identifier>(accumulator), initializer, \
storage.Create<query::Identifier>(variable), list, expr)

18
tests/unit/query_expression_evaluator.cpp
View File

@ -1221,6 +1221,24 @@ TEST(ExpressionEvaluator, FunctionAllWhereWrongType) {
EXPECT_THROW(all->Accept(eval.eval), QueryRuntimeException);
}
TEST(ExpressionEvaluator, FunctionReduce) {
AstTreeStorage storage;
auto *ident_sum = IDENT("sum");
auto *ident_x = IDENT("x");
auto *reduce = REDUCE("sum", LITERAL(0), "x", LIST(LITERAL(1), LITERAL(2)),
ADD(ident_sum, ident_x));
NoContextExpressionEvaluator eval;
const auto sum_sym = eval.symbol_table.CreateSymbol("sum", true);
eval.symbol_table[*reduce->accumulator_] = sum_sym;
eval.symbol_table[*ident_sum] = sum_sym;
const auto x_sym = eval.symbol_table.CreateSymbol("x", true);
eval.symbol_table[*reduce->identifier_] = x_sym;
eval.symbol_table[*ident_x] = x_sym;
auto value = reduce->Accept(eval.eval);
ASSERT_EQ(value.type(), TypedValue::Type::Int);
EXPECT_EQ(value.Value<int64_t>(), 3);
}
TEST(ExpressionEvaluator, FunctionAssert) {
// Invalid calls.
ASSERT_THROW(EvaluateFunction("ASSERT", {}), QueryRuntimeException);

30
tests/unit/query_semantic.cpp
View File

@ -795,6 +795,36 @@ TEST_F(TestSymbolGenerator, WithReturnAll) {
EXPECT_NE(symbol_table.at(*all->identifier_), symbol_table.at(*ret_as_x));
}
TEST_F(TestSymbolGenerator, WithReturnReduce) {
// Test WITH 42 AS x RETURN reduce(y = 0, x IN [x] y + x) AS x, x AS y
auto *with_as_x = AS("x");
auto *list_x = IDENT("x");
auto *expr_x = IDENT("x");
auto *expr_y = IDENT("y");
auto *reduce =
REDUCE("y", LITERAL(0), "x", LIST(list_x), ADD(expr_y, expr_x));
auto *ret_as_x = AS("x");
auto *ret_x = IDENT("x");
auto *ret_as_y = AS("y");
auto query = QUERY(SINGLE_QUERY(WITH(LITERAL(42), with_as_x),
RETURN(reduce, ret_as_x, ret_x, ret_as_y)));
query->Accept(symbol_generator);
// Symbols for `WITH .. AS x`, `REDUCE(y, x ...)`, `REDUCE(...) AS x` and `AS
// y`.
EXPECT_EQ(symbol_table.max_position(), 5);
// Check `WITH .. AS x` is the same as `[x]` and `RETURN ... x AS y`
EXPECT_EQ(symbol_table.at(*with_as_x), symbol_table.at(*list_x));
EXPECT_EQ(symbol_table.at(*with_as_x), symbol_table.at(*ret_x));
EXPECT_NE(symbol_table.at(*with_as_x), symbol_table.at(*reduce->identifier_));
EXPECT_NE(symbol_table.at(*with_as_x), symbol_table.at(*ret_as_x));
// Check `REDUCE(y, x ...)` is only equal to `y + x`
EXPECT_EQ(symbol_table.at(*reduce->identifier_), symbol_table.at(*expr_x));
EXPECT_NE(symbol_table.at(*reduce->identifier_), symbol_table.at(*ret_as_x));
EXPECT_EQ(symbol_table.at(*reduce->accumulator_), symbol_table.at(*expr_y));
EXPECT_NE(symbol_table.at(*reduce->accumulator_), symbol_table.at(*ret_as_y));
}
TEST_F(TestSymbolGenerator, MatchBfsReturn) {
// Test MATCH (n) -[r *bfs..n.prop] (r, n | r.prop)]-> (m) RETURN r AS r
auto prop = dba.Property("prop");