Add REDUCE function to openCypher

Reviewers: florijan, msantl

Reviewed By: florijan

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1175

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Next Release
 
 * Snapshot format changed (not backward compatible).
+* `reduce` function added.
 
 ## v0.9.0
 

docs/user_technical/open-cypher.md

@@ -550,6 +550,7 @@ functions.
  `endsWith`      | Check if the first argument ends with the second.
  `contains`      | Check if the first argument has an element which is equal to the second argument.
  `all`           | Check if all elements of a list satisfy a predicate.<br/>The syntax is: `all(variable IN list WHERE predicate)`.<br/> NOTE: Whenever possible, use Memgraph's lambda functions when [matching](#filtering-variable-length-paths) instead.
+ `reduce`        | Accumulate list elements into a single result by applying an expression. The syntax is:<br/>`reduce(accumulator = initial_value, variable IN list | expression)`.
  `assert`        | Raises an exception reported to the client if the given argument is not `true`.
  `counter`       | Generates integers that are guaranteed to be unique on the database level, for the given counter name.
  `counterSet`    | Sets the counter with the given name to the given value.

src/query/frontend/ast/ast.cpp

@@ -76,6 +76,7 @@ BOOST_CLASS_EXPORT_IMPLEMENT(query::PropertyLookup);
 BOOST_CLASS_EXPORT_IMPLEMENT(query::LabelsTest);
 BOOST_CLASS_EXPORT_IMPLEMENT(query::Aggregation);
 BOOST_CLASS_EXPORT_IMPLEMENT(query::Function);
+BOOST_CLASS_EXPORT_IMPLEMENT(query::Reduce);
 BOOST_CLASS_EXPORT_IMPLEMENT(query::All);
 BOOST_CLASS_EXPORT_IMPLEMENT(query::ParameterLookup);
 BOOST_CLASS_EXPORT_IMPLEMENT(query::Create);

src/query/frontend/ast/ast.hpp

@@ -1391,6 +1391,81 @@ class Aggregation : public BinaryOperator {
                                                         const unsigned int);
 };
 
+class Reduce : public Expression {
+  friend class AstTreeStorage;
+
+ public:
+  DEFVISITABLE(TreeVisitor<TypedValue>);
+  bool Accept(HierarchicalTreeVisitor &visitor) override {
+    if (visitor.PreVisit(*this)) {
+      accumulator_->Accept(visitor) && initializer_->Accept(visitor) &&
+          identifier_->Accept(visitor) && list_->Accept(visitor) &&
+          expression_->Accept(visitor);
+    }
+    return visitor.PostVisit(*this);
+  }
+
+  Reduce *Clone(AstTreeStorage &storage) const override {
+    return storage.Create<Reduce>(
+        accumulator_->Clone(storage), initializer_->Clone(storage),
+        identifier_->Clone(storage), list_->Clone(storage),
+        expression_->Clone(storage));
+  }
+  // None of these should be nullptr after construction.
+
+  /// Identifier for the accumulating variable
+  Identifier *accumulator_ = nullptr;
+  /// Expression which produces the initial accumulator value.
+  Expression *initializer_ = nullptr;
+  /// Identifier for the list element.
+  Identifier *identifier_ = nullptr;
+  /// Expression which produces a list which will be reduced.
+  Expression *list_ = nullptr;
+  /// Expression which does the reduction, i.e. produces the new accumulator
+  /// value.
+  Expression *expression_ = nullptr;
+
+ protected:
+  Reduce(int uid, Identifier *accumulator, Expression *initializer,
+         Identifier *identifier, Expression *list, Expression *expression)
+      : Expression(uid),
+        accumulator_(accumulator),
+        initializer_(initializer),
+        identifier_(identifier),
+        list_(list),
+        expression_(expression) {}
+
+ private:
+  friend class boost::serialization::access;
+
+  BOOST_SERIALIZATION_SPLIT_MEMBER();
+
+  template <class TArchive>
+  void save(TArchive &ar, const unsigned int) const {
+    ar << boost::serialization::base_object<Expression>(*this);
+    SavePointer(ar, accumulator_);
+    SavePointer(ar, initializer_);
+    SavePointer(ar, identifier_);
+    SavePointer(ar, list_);
+    SavePointer(ar, expression_);
+  }
+
+  template <class TArchive>
+  void load(TArchive &ar, const unsigned int) {
+    ar >> boost::serialization::base_object<Expression>(*this);
+    LoadPointer(ar, accumulator_);
+    LoadPointer(ar, initializer_);
+    LoadPointer(ar, identifier_);
+    LoadPointer(ar, list_);
+    LoadPointer(ar, expression_);
+  }
+
+  template <class TArchive>
+  friend void boost::serialization::load_construct_data(TArchive &, Reduce *,
+                                                        const unsigned int);
+};
+
+// TODO: Think about representing All and Any as Reduce.
 class All : public Expression {
   friend class AstTreeStorage;
 
@@ -2847,6 +2922,8 @@ LOAD_AND_CONSTRUCT(query::LabelsTest, 0, nullptr,
 LOAD_AND_CONSTRUCT(query::Function, 0);
 LOAD_AND_CONSTRUCT(query::Aggregation, 0, nullptr, nullptr,
                    query::Aggregation::Op::COUNT);
+LOAD_AND_CONSTRUCT(query::Reduce, 0, nullptr, nullptr, nullptr, nullptr,
+                   nullptr);
 LOAD_AND_CONSTRUCT(query::All, 0, nullptr, nullptr, nullptr);
 LOAD_AND_CONSTRUCT(query::ParameterLookup, 0);
 LOAD_AND_CONSTRUCT(query::NamedExpression, 0);
@@ -2906,6 +2983,7 @@ BOOST_CLASS_EXPORT_KEY(query::PropertyLookup);
 BOOST_CLASS_EXPORT_KEY(query::LabelsTest);
 BOOST_CLASS_EXPORT_KEY(query::Aggregation);
 BOOST_CLASS_EXPORT_KEY(query::Function);
+BOOST_CLASS_EXPORT_KEY(query::Reduce);
 BOOST_CLASS_EXPORT_KEY(query::All);
 BOOST_CLASS_EXPORT_KEY(query::ParameterLookup);
 BOOST_CLASS_EXPORT_KEY(query::Create);

src/query/frontend/ast/ast_visitor.hpp

@@ -14,6 +14,7 @@ class PropertyLookup;
 class LabelsTest;
 class Aggregation;
 class Function;
+class Reduce;
 class All;
 class ParameterLookup;
 class Create;
@@ -67,8 +68,8 @@ using TreeCompositeVisitor = ::utils::CompositeVisitor<
     GreaterEqualOperator, InListOperator, ListMapIndexingOperator,
     ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator,
     IsNullOperator, ListLiteral, MapLiteral, PropertyLookup, LabelsTest,
-    Aggregation, Function, All, Create, Match, Return, With, Pattern, NodeAtom,
+    Aggregation, Function, Reduce, All, Create, Match, Return, With, Pattern,
-    EdgeAtom, Delete, Where, SetProperty, SetProperties, SetLabels,
+    NodeAtom, EdgeAtom, Delete, Where, SetProperty, SetProperties, SetLabels,
     RemoveProperty, RemoveLabels, Merge, Unwind>;
 
 using TreeLeafVisitor = ::utils::LeafVisitor<Identifier, PrimitiveLiteral,
@@ -92,9 +93,9 @@ using TreeVisitor = ::utils::Visitor<
     LessEqualOperator, GreaterEqualOperator, InListOperator,
     ListMapIndexingOperator, ListSlicingOperator, IfOperator, UnaryPlusOperator,
     UnaryMinusOperator, IsNullOperator, ListLiteral, MapLiteral, PropertyLookup,
-    LabelsTest, Aggregation, Function, All, ParameterLookup, Create, Match,
+    LabelsTest, Aggregation, Function, Reduce, All, ParameterLookup, Create,
-    Return, With, Pattern, NodeAtom, EdgeAtom, Delete, Where, SetProperty,
+    Match, Return, With, Pattern, NodeAtom, EdgeAtom, Delete, Where,
-    SetProperties, SetLabels, RemoveProperty, RemoveLabels, Merge, Unwind,
+    SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels, Merge,
-    Identifier, PrimitiveLiteral, CreateIndex>;
+    Unwind, Identifier, PrimitiveLiteral, CreateIndex>;
 
 }  // namespace query

src/query/frontend/ast/cypher_main_visitor.cpp

@@ -865,6 +865,21 @@ antlrcpp::Any CypherMainVisitor::visitAtom(CypherParser::AtomContext *ctx) {
     Where *where = ctx->filterExpression()->where()->accept(this);
     return static_cast<Expression *>(
         storage_.Create<All>(ident, list_expr, where));
+  } else if (ctx->REDUCE()) {
+    auto *accumulator = storage_.Create<Identifier>(
+        ctx->reduceExpression()->accumulator->accept(this).as<std::string>());
+    Expression *initializer = ctx->reduceExpression()->initial->accept(this);
+    auto *ident = storage_.Create<Identifier>(ctx->reduceExpression()
+                                                  ->idInColl()
+                                                  ->variable()
+                                                  ->accept(this)
+                                                  .as<std::string>());
+    Expression *list =
+        ctx->reduceExpression()->idInColl()->expression()->accept(this);
+    Expression *expr =
+        ctx->reduceExpression()->expression().back()->accept(this);
+    return static_cast<Expression *>(
+        storage_.Create<Reduce>(accumulator, initializer, ident, list, expr));
   } else if (ctx->caseExpression()) {
     return static_cast<Expression *>(ctx->caseExpression()->accept(this));
   }

src/query/frontend/opencypher/grammar/Cypher.g4

@@ -187,6 +187,7 @@ atom : literal
      | patternComprehension
      | ( FILTER SP? '(' SP? filterExpression SP? ')' )
      | ( EXTRACT SP? '(' SP? filterExpression SP? ( SP? '|' expression )? ')' )
+     | ( REDUCE SP? '(' SP? reduceExpression SP? ')' )
      | ( ALL SP? '(' SP? filterExpression SP? ')' )
      | ( ANY SP? '(' SP? filterExpression SP? ')' )
      | ( NONE SP? '(' SP? filterExpression SP? ')' )
@@ -226,6 +227,8 @@ relationshipsPattern : nodePattern ( SP? patternElementChain )+ ;
 
 filterExpression : idInColl ( SP? where )? ;
 
+reduceExpression : accumulator=variable SP? '=' SP? initial=expression SP? ',' SP? idInColl SP? '|' SP? expression ;
+
 idInColl : variable SP IN SP expression ;
 
 functionInvocation : functionName SP? '(' SP? ( DISTINCT SP? )? ( expression SP? ( ',' SP? expression SP? )* )? ')' ;
@@ -327,6 +330,7 @@ symbolicName : UnescapedSymbolicName
              | EscapedSymbolicName
              | UNION
              | ALL
+             | REDUCE
              | OPTIONAL
              | MATCH
              | UNWIND
@@ -380,6 +384,8 @@ UNION : ( 'U' | 'u' ) ( 'N' | 'n' ) ( 'I' | 'i' ) ( 'O' | 'o' ) ( 'N' | 'n' )  ;
 
 ALL : ( 'A' | 'a' ) ( 'L' | 'l' ) ( 'L' | 'l' )  ;
 
+REDUCE : ( 'R' | 'r' ) ( 'E' | 'e' ) ( 'D' | 'd' ) ( 'U' | 'u' ) ( 'C' | 'c' ) ( 'E' | 'e' ) ;
+
 OPTIONAL : ( 'O' | 'o' ) ( 'P' | 'p' ) ( 'T' | 't' ) ( 'I' | 'i' ) ( 'O' | 'o' ) ( 'N' | 'n' ) ( 'A' | 'a' ) ( 'L' | 'l' )  ;
 
 MATCH : ( 'M' | 'm' ) ( 'A' | 'a' ) ( 'T' | 't' ) ( 'C' | 'c' ) ( 'H' | 'h' )  ;

src/query/frontend/semantic/symbol_generator.cpp

@@ -333,6 +333,15 @@ bool SymbolGenerator::PreVisit(All &all) {
   return false;
 }
 
+bool SymbolGenerator::PreVisit(Reduce &reduce) {
+  reduce.initializer_->Accept(*this);
+  reduce.list_->Accept(*this);
+  VisitWithIdentifiers(*reduce.expression_,
+                       {reduce.accumulator_, reduce.identifier_});
+  return false;
+}
+
+
 // Pattern and its subparts.
 
 bool SymbolGenerator::PreVisit(Pattern &pattern) {

src/query/frontend/semantic/symbol_generator.hpp

@@ -57,6 +57,7 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
   bool PreVisit(IfOperator &) override;
   bool PostVisit(IfOperator &) override;
   bool PreVisit(All &) override;
+  bool PreVisit(Reduce &) override;
 
   // Pattern and its subparts.
   bool PreVisit(Pattern &) override;

src/query/frontend/stripped_lexer_constants.hpp

@@ -86,7 +86,8 @@ const trie::Trie kKeywords = {
     "where",   "or",    "xor",       "and",    "not",        "in",
     "starts",  "ends",  "contains",  "is",     "null",       "case",
     "when",    "then",  "else",      "end",    "count",      "filter",
-    "extract", "any",   "none",      "single", "true",       "false"};
+    "extract", "any",   "none",      "single", "true",       "false",
+    "reduce"};
 
 // Unicode codepoints that are allowed at the start of the unescaped name.
 const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(std::string(

src/query/interpret/eval.hpp

@@ -354,6 +354,27 @@ class ExpressionEvaluator : public TreeVisitor<TypedValue> {
     return function.function()(arguments, db_accessor_);
   }
 
+  TypedValue Visit(Reduce &reduce) override {
+    auto list_value = reduce.list_->Accept(*this);
+    if (list_value.IsNull()) {
+      return TypedValue::Null;
+    }
+    if (list_value.type() != TypedValue::Type::List) {
+      throw QueryRuntimeException("'REDUCE' expected a list, but got {}",
+                                  list_value.type());
+    }
+    const auto &list = list_value.Value<std::vector<TypedValue>>();
+    const auto &element_symbol = symbol_table_.at(*reduce.identifier_);
+    const auto &accumulator_symbol = symbol_table_.at(*reduce.accumulator_);
+    auto accumulator = reduce.initializer_->Accept(*this);
+    for (const auto &element : list) {
+      frame_[accumulator_symbol] = accumulator;
+      frame_[element_symbol] = element;
+      accumulator = reduce.expression_->Accept(*this);
+    }
+    return accumulator;
+  }
+
   TypedValue Visit(All &all) override {
     auto list_value = all.list_expression_->Accept(*this);
     if (list_value.IsNull()) {

src/query/plan/preprocess.hpp

@@ -30,6 +30,15 @@ class UsedSymbolsCollector : public HierarchicalTreeVisitor {
     return true;
   }
 
+  bool PostVisit(Reduce &reduce) override {
+    // Remove the symbols bound by reduce, because we are only interested
+    // in free (unbound) symbols.
+    symbols_.erase(symbol_table_.at(*reduce.accumulator_));
+    symbols_.erase(symbol_table_.at(*reduce.identifier_));
+    return true;
+  }
+
+
   bool Visit(Identifier &ident) override {
     symbols_.insert(symbol_table_.at(ident));
     return true;

src/query/plan/rule_based_planner.cpp

@@ -210,6 +210,23 @@ class ReturnBodyContext : public HierarchicalTreeVisitor {
     return true;
   }
 
+  bool PostVisit(Reduce &reduce) override {
+    // Remove the symbols bound by reduce, because we are only interested
+    // in free (unbound) symbols.
+    used_symbols_.erase(symbol_table_.at(*reduce.accumulator_));
+    used_symbols_.erase(symbol_table_.at(*reduce.identifier_));
+    DCHECK(has_aggregation_.size() >= 5U)
+        << "Expected 5 has_aggregation_ flags for REDUCE arguments";
+    bool has_aggr = false;
+    for (int i = 0; i < 5; ++i) {
+      has_aggr = has_aggr || has_aggregation_.back();
+      has_aggregation_.pop_back();
+    }
+    has_aggregation_.emplace_back(has_aggr);
+    return true;
+  }
+
+
   bool Visit(Identifier &ident) override {
     const auto &symbol = symbol_table_.at(ident);
     if (!utils::Contains(output_symbols_, symbol)) {

tests/qa/tck_engine/tests/memgraph_V1/features/functions.feature

@@ -678,6 +678,30 @@ Feature: Functions
             """
         Then an error should be raised
 
+    Scenario: Reduce test 01:
+        When executing query:
+            """
+            RETURN reduce(a = true, x IN [1, 2, '3'] | a AND x < 2) AS a
+            """
+        Then the result should be:
+            | a     |
+            | false |
+
+    Scenario: Reduce test 02:
+        When executing query:
+            """
+            RETURN reduce(s = 0, x IN [1, 2, 3] | s + x) AS s
+            """
+        Then the result should be:
+            | s |
+            | 6 |
+
+    Scenario: Reduce test 03:
+        When executing query:
+            """
+            RETURN reduce(a = true, x IN [true, true, '3'] | a AND x) AS a
+            """
+        Then an error should be raised
 
     Scenario: Assert test fail, no message:
         Given an empty graph

tests/unit/cypher_main_visitor.cpp

@@ -1563,6 +1563,27 @@ TYPED_TEST(CypherMainVisitorTest, ReturnAll) {
   EXPECT_TRUE(eq);
 }
 
+TYPED_TEST(CypherMainVisitorTest, ReturnReduce) {
+  TypeParam ast_generator("RETURN reduce(sum = 0, x IN [1,2,3] | sum + x)");
+  auto *query = ast_generator.query_;
+  ASSERT_TRUE(query->single_query_);
+  auto *single_query = query->single_query_;
+  ASSERT_EQ(single_query->clauses_.size(), 1U);
+  auto *ret = dynamic_cast<Return *>(single_query->clauses_[0]);
+  ASSERT_TRUE(ret);
+  ASSERT_EQ(ret->body_.named_expressions.size(), 1U);
+  auto *reduce =
+      dynamic_cast<Reduce *>(ret->body_.named_expressions[0]->expression_);
+  ASSERT_TRUE(reduce);
+  EXPECT_EQ(reduce->accumulator_->name_, "sum");
+  CheckLiteral(ast_generator.context_, reduce->initializer_, 0);
+  EXPECT_EQ(reduce->identifier_->name_, "x");
+  auto *list_literal = dynamic_cast<ListLiteral *>(reduce->list_);
+  EXPECT_TRUE(list_literal);
+  auto *add = dynamic_cast<AdditionOperator *>(reduce->expression_);
+  EXPECT_TRUE(add);
+}
+
 TYPED_TEST(CypherMainVisitorTest, MatchBfsReturn) {
   TypeParam ast_generator(
       "MATCH (n) -[r:type1|type2 *bfs..10 (e, n|e.prop = 42)]-> (m) RETURN r");

tests/unit/query_common.hpp

@@ -597,3 +597,7 @@ auto GetMerge(AstTreeStorage &storage, Pattern *pattern, OnMatch on_match,
 #define ALL(variable, list, where)                                        \
   storage.Create<query::All>(storage.Create<query::Identifier>(variable), \
                              list, where)
+#define REDUCE(accumulator, initializer, variable, list, expr)     \
+  storage.Create<query::Reduce>(                                   \
+      storage.Create<query::Identifier>(accumulator), initializer, \
+      storage.Create<query::Identifier>(variable), list, expr)

tests/unit/query_expression_evaluator.cpp

@@ -1221,6 +1221,24 @@ TEST(ExpressionEvaluator, FunctionAllWhereWrongType) {
   EXPECT_THROW(all->Accept(eval.eval), QueryRuntimeException);
 }
 
+TEST(ExpressionEvaluator, FunctionReduce) {
+  AstTreeStorage storage;
+  auto *ident_sum = IDENT("sum");
+  auto *ident_x = IDENT("x");
+  auto *reduce = REDUCE("sum", LITERAL(0), "x", LIST(LITERAL(1), LITERAL(2)),
+                        ADD(ident_sum, ident_x));
+  NoContextExpressionEvaluator eval;
+  const auto sum_sym = eval.symbol_table.CreateSymbol("sum", true);
+  eval.symbol_table[*reduce->accumulator_] = sum_sym;
+  eval.symbol_table[*ident_sum] = sum_sym;
+  const auto x_sym = eval.symbol_table.CreateSymbol("x", true);
+  eval.symbol_table[*reduce->identifier_] = x_sym;
+  eval.symbol_table[*ident_x] = x_sym;
+  auto value = reduce->Accept(eval.eval);
+  ASSERT_EQ(value.type(), TypedValue::Type::Int);
+  EXPECT_EQ(value.Value<int64_t>(), 3);
+}
+
 TEST(ExpressionEvaluator, FunctionAssert) {
   // Invalid calls.
   ASSERT_THROW(EvaluateFunction("ASSERT", {}), QueryRuntimeException);

tests/unit/query_semantic.cpp

@@ -795,6 +795,36 @@ TEST_F(TestSymbolGenerator, WithReturnAll) {
   EXPECT_NE(symbol_table.at(*all->identifier_), symbol_table.at(*ret_as_x));
 }
 
+TEST_F(TestSymbolGenerator, WithReturnReduce) {
+  // Test WITH 42 AS x RETURN reduce(y = 0, x IN [x] y + x) AS x, x AS y
+  auto *with_as_x = AS("x");
+  auto *list_x = IDENT("x");
+  auto *expr_x = IDENT("x");
+  auto *expr_y = IDENT("y");
+  auto *reduce =
+      REDUCE("y", LITERAL(0), "x", LIST(list_x), ADD(expr_y, expr_x));
+  auto *ret_as_x = AS("x");
+  auto *ret_x = IDENT("x");
+  auto *ret_as_y = AS("y");
+  auto query = QUERY(SINGLE_QUERY(WITH(LITERAL(42), with_as_x),
+                                  RETURN(reduce, ret_as_x, ret_x, ret_as_y)));
+  query->Accept(symbol_generator);
+  // Symbols for `WITH .. AS x`, `REDUCE(y, x ...)`, `REDUCE(...) AS x` and `AS
+  // y`.
+  EXPECT_EQ(symbol_table.max_position(), 5);
+  // Check `WITH .. AS x` is the same as `[x]` and `RETURN ... x AS y`
+  EXPECT_EQ(symbol_table.at(*with_as_x), symbol_table.at(*list_x));
+  EXPECT_EQ(symbol_table.at(*with_as_x), symbol_table.at(*ret_x));
+  EXPECT_NE(symbol_table.at(*with_as_x), symbol_table.at(*reduce->identifier_));
+  EXPECT_NE(symbol_table.at(*with_as_x), symbol_table.at(*ret_as_x));
+  // Check `REDUCE(y, x ...)` is only equal to `y + x`
+  EXPECT_EQ(symbol_table.at(*reduce->identifier_), symbol_table.at(*expr_x));
+  EXPECT_NE(symbol_table.at(*reduce->identifier_), symbol_table.at(*ret_as_x));
+  EXPECT_EQ(symbol_table.at(*reduce->accumulator_), symbol_table.at(*expr_y));
+  EXPECT_NE(symbol_table.at(*reduce->accumulator_), symbol_table.at(*ret_as_y));
+}
+
+
 TEST_F(TestSymbolGenerator, MatchBfsReturn) {
   // Test MATCH (n) -[r *bfs..n.prop] (r, n | r.prop)]-> (m) RETURN r AS r
   auto prop = dba.Property("prop");