tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remember token positions for literals

Browse Source 
Summary: Remove yaml

Reviewers: buda

Reviewed By: buda

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D466
This commit is contained in:
Mislav Bradac 2017-06-13 16:25:08 +02:00
parent c8e25987b8
commit c56eb4310e
19 changed files with 84 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CMakeLists.txt
View File

@ -110,11 +110,6 @@ add_subdirectory(libs)
# fmt
set(fmt_source_dir ${libs_dir}/fmt)
set(fmt_static_lib ${fmt_source_dir}/fmt/libfmt.a)
# yaml-cpp
# We no longer use yaml for configuration reading, maybe this dependancy should
# be removed. It is only used in tests/benchmark/query/strip/stripper.cpp
set(yaml_source_dir ${libs_dir}/yaml-cpp)
set(yaml_include_dir ${yaml_source_dir}/include)
# prepare template and destination folders for query engine (tests)
# and memgraph server binary
@ -158,7 +153,7 @@ if(CLANG_TIDY)
-config=''
--
-std=c++1y
-I${CMAKE_SOURCE_DIR}/include -I${fmt_source_dir} -I${yaml_include_dir}
-I${CMAKE_SOURCE_DIR}/include -I${fmt_source_dir}
)
endif()
# -----------------------------------------------------------------------------
@ -276,7 +271,6 @@ message(STATUS "Generate coverage from unit tests: ${TEST_COVERAGE}")
include_directories(${src_dir})
include_directories(${build_include_dir})
include_directories(SYSTEM ${fmt_source_dir})
include_directories(SYSTEM ${yaml_include_dir})
include_directories(SYSTEM ${GTEST_INCLUDE_DIRS} ${GMOCK_INCLUDE_DIRS})
include_directories(SYSTEM ${CMAKE_SOURCE_DIR}/libs)
# -----------------------------------------------------------------------------
@ -351,7 +345,7 @@ set(memgraph_src_files
${src_dir}/database/graph_db_accessor.cpp
${src_dir}/data_structures/concurrent/skiplist_gc.cpp
${src_dir}/query/engine.cpp
${src_dir}/query/stripped.cpp
${src_dir}/query/frontend/stripped.cpp
${src_dir}/query/common.cpp
${src_dir}/query/console.cpp
${src_dir}/query/frontend/ast/ast.cpp
@ -367,7 +361,7 @@ set(memgraph_src_files
# -----------------------------------------------------------------------------
# memgraph_lib and memgraph_pic depend on these libraries
set(MEMGRAPH_ALL_LIBS gflags stdc++fs Threads::Threads fmt yaml-cpp antlr_opencypher_parser_lib dl)
set(MEMGRAPH_ALL_LIBS gflags stdc++fs Threads::Threads fmt antlr_opencypher_parser_lib dl)
if (READLINE_FOUND)
list(APPEND MEMGRAPH_ALL_LIBS ${READLINE_LIBRARY})
endif()

7
libs/CMakeLists.txt
View File

@ -21,13 +21,6 @@ set(GFLAGS_BUILD_gflags_nothreads_LIB OFF)
set(GFLAGS_BUILD_gflags_LIB ON)
add_subdirectory(gflags)
# setup yaml cpp
# disable tests because yaml doesn't have MASTER_PROJECT flag like fmt has
# to override an option use option :)
option(YAML_CPP_BUILD_TOOLS "" OFF)
add_subdirectory(yaml-cpp)
# setup cppitertools
# CLion compatiblity; the target won't be built
file(GLOB __CPPITERTOOLS_SOURCES __main.cpp

9
libs/setup.sh
View File

@ -47,15 +47,6 @@ cd googletest
git checkout ${googletest_tag}
cd ..
# yaml-cpp
git clone https://github.com/jbeder/yaml-cpp
yaml_cpp_tag="519d33fea3fbcbe7e1f89f97ee0fa539cec33eb7" # master 18 Aug 2016
# because a bug with link process had been fixed somewhen between
# this commit and v0.5.3
cd yaml-cpp
git checkout ${yaml_cpp_tag}
cd ..
# lcov-to-coberatura-xml
git clone https://github.com/eriwen/lcov-to-cobertura-xml.git
lcov_to_xml_tag="59584761cb5da4687693faec05bf3e2b74e9dde9" # Dec 6, 2016

2
src/copy_hardcoded_queries.cpp
View File

@ -7,7 +7,7 @@ namespace fs = std::experimental::filesystem;
#include "logging/logger.hpp"
#include "logging/streams/stdout.hpp"
#include "query/stripped.hpp"
#include "query/frontend/stripped.hpp"
#include "utils/command_line/arguments.hpp"
#include "utils/exceptions.hpp"
#include "utils/file.hpp"

9
src/query/frontend/ast/ast.hpp
View File

@ -529,15 +529,22 @@ class PrimitiveLiteral : public BaseLiteral {
DEFVISITABLE(HierarchicalTreeVisitor);
PrimitiveLiteral *Clone(AstTreeStorage &storage) const override {
return storage.Create<PrimitiveLiteral>(value_);
return storage.Create<PrimitiveLiteral>(value_, token_position_);
}
TypedValue value_;
// This field contains token position of literal used to create
// PrimitiveLiteral object. If PrimitiveLiteral object is not created from
// query leave its value at -1.
int token_position_ = -1;
protected:
PrimitiveLiteral(int uid) : BaseLiteral(uid) {}
template <typename T>
PrimitiveLiteral(int uid, T value) : BaseLiteral(uid), value_(value) {}
template <typename T>
PrimitiveLiteral(int uid, T value, int token_position)
: BaseLiteral(uid), value_(value), token_position_(token_position) {}
};
class ListLiteral : public BaseLiteral {

10
src/query/frontend/ast/cypher_main_visitor.cpp
View File

@ -733,18 +733,20 @@ antlrcpp::Any CypherMainVisitor::visitAtom(CypherParser::AtomContext *ctx) {
antlrcpp::Any CypherMainVisitor::visitLiteral(
CypherParser::LiteralContext *ctx) {
int token_position = ctx->getStart()->getTokenIndex();
if (ctx->CYPHERNULL()) {
return static_cast<BaseLiteral *>(
storage_.Create<PrimitiveLiteral>(TypedValue::Null));
storage_.Create<PrimitiveLiteral>(TypedValue::Null, token_position));
} else if (ctx->StringLiteral()) {
return static_cast<BaseLiteral *>(storage_.Create<PrimitiveLiteral>(
visitStringLiteral(ctx->StringLiteral()->getText()).as<std::string>()));
visitStringLiteral(ctx->StringLiteral()->getText()).as<std::string>(),
token_position));
} else if (ctx->booleanLiteral()) {
return static_cast<BaseLiteral *>(storage_.Create<PrimitiveLiteral>(
ctx->booleanLiteral()->accept(this).as<bool>()));
ctx->booleanLiteral()->accept(this).as<bool>(), token_position));
} else if (ctx->numberLiteral()) {
return static_cast<BaseLiteral *>(storage_.Create<PrimitiveLiteral>(
ctx->numberLiteral()->accept(this).as<TypedValue>()));
ctx->numberLiteral()->accept(this).as<TypedValue>(), token_position));
} else if (ctx->listLiteral()) {
return static_cast<BaseLiteral *>(storage_.Create<ListLiteral>(
ctx->listLiteral()->accept(this).as<std::vector<Expression *>>()));

3
src/query/stripped.cpp → src/query/frontend/stripped.cpp
View File

@ -1,4 +1,4 @@
#include "query/stripped.hpp"
#include "query/frontend/stripped.hpp"
#include <iostream>
#include <string>
@ -36,6 +36,7 @@ StrippedQuery::StrippedQuery(const std::string &query)
// literal, appends is to the the stripped_query and adds the passed
// value to stripped args.
auto replace_stripped = [this, &token_strings](const TypedValue &value) {
// const std::string &new_value) {
const auto &stripped_name = parameters_.Add(value);
token_strings.push_back("$" + stripped_name);
};

2
src/query/stripped.hpp → src/query/frontend/stripped.hpp
View File

@ -3,7 +3,7 @@
#include <map>
#include "logging/loggable.hpp"
#include "parameters.hpp"
#include "query/parameters.hpp"
#include "storage/property_value_store.hpp"
#include "utils/assert.hpp"
#include "utils/hashing/fnv.hpp"

2
src/query/interpreter.hpp
View File

@ -137,7 +137,7 @@ class Interpreter : public Loggable {
}
private:
// ConcurrentMap<HashType, std::unique_ptr<QueryPlanLib>> query_plans;
// ConcurrentMap<HashType, CachedAst> ast_cache_;
};
} // namespace query

2
src/query/plan_interface.hpp
View File

@ -2,7 +2,7 @@
#include "database/graph_db_accessor.hpp"
#include "query/stripped.hpp"
#include "query/frontend/stripped.hpp"
/**
* @class PlanInterface

20
tests/benchmark/query/strip/stripper.cpp
View File

@ -1,10 +1,14 @@
#define LOG_NO_INFO 1
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include "benchmark/benchmark_api.h"
#include "logging/default.hpp"
#include "logging/streams/stdout.hpp"
#include "query/stripped.hpp"
#include "yaml-cpp/yaml.h"
#include "query/frontend/stripped.hpp"
auto BM_Strip = [](benchmark::State &state, auto &function, std::string query) {
while (state.KeepRunning()) {
@ -15,19 +19,19 @@ auto BM_Strip = [](benchmark::State &state, auto &function, std::string query) {
state.SetComplexityN(state.range(0));
};
int main(int argc, char **argv) {
int main(int argc, char *argv[]) {
logging::init_async();
logging::log->pipe(std::make_unique<Stdout>());
YAML::Node dataset = YAML::LoadFile(
"../../tests/data/cypher_queries/stripper/query_dict.yaml");
auto preprocess = [](const std::string &query) {
return query::StrippedQuery(query);
};
auto tests = dataset["benchmark_queries"].as<std::vector<std::string>>();
for (auto &test : tests) {
std::string path = "../../tests/data/cypher_queries/stripper/query_dict.yaml";
std::fstream queries_file(path);
std::string test;
while (std::getline(queries_file, test)) {
benchmark::RegisterBenchmark(test.c_str(), BM_Strip, preprocess, test)
->Range(1, 1)
->Complexity(benchmark::oN);

63
tests/data/cypher_queries/stripper/query_dict.yaml
View File

@ -1,42 +1,21 @@
benchmark_queries:
- "MATCH (a) RETURN size(collect(a))"
- "CREATE (a:L), (b1), (b2) CREATE (a)-[:A]->(b1), (a)-[:A]->(b2)"
- "MATCH (a:L)-[rel]->(b) RETURN a, count(*)"
- "CREATE ({division: 'Sweden'})"
- "MATCH (n) RETURN n.division, count(*) ORDER BY count(*) DESC, n.division ASC"
- "UNWIND ['a', 'b', 'B', null, 'abc', 'abc1'] AS i RETURN max(i)"
- "CREATE ({created: true})"
- "MATCH (a)-[r]-(b) DELETE r, a, b RETURN count(*) AS c"
- "MATCH (u:User) WITH {key: u} AS nodes DELETE nodes.key"
- "CREATE ()-[:T {id: 42, alive: true, name: kifla, height: 4.2}]->()"
- "MATCH p = ()-[r:T]-() WHERE r.id = 42 DELETE r"
- "UNWIND range(0, 1000) AS i CREATE (:A {id: i}) MERGE (:B {id: i % 10})"
- "MATCH (n) WHERE NOT(n.name = 'apa' AND false) RETURN n"
- "CREATE ()-[:REL {property1: 12, property2: 24}]->()"
- "MATCH (n:A) WHERE n.name = 'Andres' SET n.name = 'Michael' RETURN n"
- "MATCH (n:A) SET (n).name = 'memgraph' RETURN n"
- "CREATE (a {foo: [1, 2, 3]}) SET a.foo = a.foo + [4, 5] RETURN a.foo"
- "MATCH (n:X {foo: 'A'}) SET n = {foo: 'B', baz: 'C'} RETURN n"
- "MATCH (n:X {foo: 'A'}) SET n += {foo: null} RETURN n"
- "MATCH (n) WITH n LIMIT toInteger(ceil(1.7)) RETURN count(*) AS count"
- "MATCH (a:A), (b:B) MERGE (a)-[r:TYPE]->(b) ON CREATE SET r.name = 'Lola' RETURN count(r)"
MATCH (a) RETURN size(collect(a))
CREATE (a:L), (b1), (b2) CREATE (a)-[:A]->(b1), (a)-[:A]->(b2)
MATCH (a:L)-[rel]->(b) RETURN a, count(*)
CREATE ({division: 'Sweden'})
MATCH (n) RETURN n.division, count(*) ORDER BY count(*) DESC, n.division ASC
UNWIND ['a', 'b', 'B', null, 'abc', 'abc1'] AS i RETURN max(i)
CREATE ({created: true})
MATCH (a)-[r]-(b) DELETE r, a, b RETURN count(*) AS c
MATCH (u:User) WITH {key: u} AS nodes DELETE nodes.key
CREATE ()-[:T {id: 42, alive: true, name: kifla, height: 4.2}]->()
MATCH p = ()-[r:T]-() WHERE r.id = 42 DELETE r
UNWIND range(0, 1000) AS i CREATE (:A {id: i}) MERGE (:B {id: i % 10})
MATCH (n) WHERE NOT(n.name = 'apa' AND false) RETURN n
CREATE ()-[:REL {property1: 12, property2: 24}]->()
MATCH (n:A) WHERE n.name = 'Andres' SET n.name = 'Michael' RETURN n
MATCH (n:A) SET (n).name = 'memgraph' RETURN n
CREATE (a {foo: [1, 2, 3]}) SET a.foo = a.foo + [4, 5] RETURN a.foo
MATCH (n:X {foo: 'A'}) SET n = {foo: 'B', baz: 'C'} RETURN n
MATCH (n:X {foo: 'A'}) SET n += {foo: null} RETURN n
MATCH (n) WITH n LIMIT toInteger(ceil(1.7)) RETURN count(*) AS count
MATCH (a:A), (b:B) MERGE (a)-[r:TYPE]->(b) ON CREATE SET r.name = 'Lola' RETURN count(r)

2
tests/integration/hardcoded_query/delete_all.cpp
View File

@ -3,7 +3,7 @@
#include "query/parameters.hpp"
#include "query/plan_interface.hpp"
#include "query/stripped.hpp"
#include "query/frontend/stripped.hpp"
#include "query/typed_value.hpp"
#include "using.hpp"

2
tests/integration/hardcoded_query/match_profile_garment_score.cpp
View File

@ -3,7 +3,7 @@
#include "query/parameters.hpp"
#include "query/plan_interface.hpp"
#include "query/stripped.hpp"
#include "query/frontend/stripped.hpp"
#include "query/typed_value.hpp"
#include "storage/edge_accessor.hpp"
#include "storage/vertex_accessor.hpp"

2
tests/integration/query_engine_common.hpp
View File

@ -8,7 +8,7 @@ namespace fs = std::experimental::filesystem;
#include "logging/default.hpp"
#include "logging/streams/stdout.cpp"
#include "query/engine.hpp"
#include "query/stripped.hpp"
#include "query/frontend/stripped.hpp"
#include "stream/print_record_stream.hpp"
#include "utils/command_line/arguments.hpp"
#include "utils/file.hpp"

2
tests/manual/query_hash.cpp
View File

@ -3,7 +3,7 @@
#include "logging/default.hpp"
#include "logging/streams/stdout.hpp"
#include "query/stripped.hpp"
#include "query/frontend/stripped.hpp"
#include "utils/command_line/arguments.hpp"
#include "utils/type_discovery.hpp"

2
tests/manual/query_stripper_timing.cpp
View File

@ -6,7 +6,7 @@
#include <ctime>
#include <iostream>
#include "query/stripped.hpp"
#include "query/frontend/stripped.hpp"
int main(int argc, const char **a) {
if (argc < 2) {

33
tests/unit/cypher_main_visitor.cpp
View File

@ -228,7 +228,8 @@ TYPED_TEST(CypherMainVisitorTest, IntegerLiteral) {
auto *literal = dynamic_cast<PrimitiveLiteral *>(
return_clause->body_.named_expressions[0]->expression_);
ASSERT_TRUE(literal);
ASSERT_EQ(literal->value_.Value<int64_t>(), 42);
EXPECT_EQ(literal->value_.Value<int64_t>(), 42);
EXPECT_EQ(literal->token_position_, 2);
}
TYPED_TEST(CypherMainVisitorTest, IntegerLiteralTooLarge) {
@ -243,7 +244,8 @@ TYPED_TEST(CypherMainVisitorTest, BooleanLiteralTrue) {
auto *literal = dynamic_cast<PrimitiveLiteral *>(
return_clause->body_.named_expressions[0]->expression_);
ASSERT_TRUE(literal);
ASSERT_EQ(literal->value_.Value<bool>(), true);
EXPECT_EQ(literal->value_.Value<bool>(), true);
EXPECT_EQ(literal->token_position_, 2);
}
TYPED_TEST(CypherMainVisitorTest, BooleanLiteralFalse) {
@ -253,7 +255,8 @@ TYPED_TEST(CypherMainVisitorTest, BooleanLiteralFalse) {
auto *literal = dynamic_cast<PrimitiveLiteral *>(
return_clause->body_.named_expressions[0]->expression_);
ASSERT_TRUE(literal);
ASSERT_EQ(literal->value_.Value<bool>(), false);
EXPECT_EQ(literal->value_.Value<bool>(), false);
EXPECT_EQ(literal->token_position_, 2);
}
TYPED_TEST(CypherMainVisitorTest, NullLiteral) {
@ -263,7 +266,8 @@ TYPED_TEST(CypherMainVisitorTest, NullLiteral) {
auto *literal = dynamic_cast<PrimitiveLiteral *>(
return_clause->body_.named_expressions[0]->expression_);
ASSERT_TRUE(literal);
ASSERT_EQ(literal->value_.type(), TypedValue::Type::Null);
EXPECT_EQ(literal->value_.type(), TypedValue::Type::Null);
EXPECT_EQ(literal->token_position_, 2);
}
TYPED_TEST(CypherMainVisitorTest, ParenthesizedExpression) {
@ -602,7 +606,8 @@ TYPED_TEST(CypherMainVisitorTest, StringLiteralDoubleQuotes) {
auto *literal = dynamic_cast<PrimitiveLiteral *>(
return_clause->body_.named_expressions[0]->expression_);
ASSERT_TRUE(literal);
ASSERT_EQ(literal->value_.Value<std::string>(), "mi'rko");
EXPECT_EQ(literal->value_.Value<std::string>(), "mi'rko");
EXPECT_EQ(literal->token_position_, 2);
}
TYPED_TEST(CypherMainVisitorTest, StringLiteralSingleQuotes) {
@ -612,7 +617,8 @@ TYPED_TEST(CypherMainVisitorTest, StringLiteralSingleQuotes) {
auto *literal = dynamic_cast<PrimitiveLiteral *>(
return_clause->body_.named_expressions[0]->expression_);
ASSERT_TRUE(literal);
ASSERT_EQ(literal->value_.Value<std::string>(), "mi\"rko");
EXPECT_EQ(literal->value_.Value<std::string>(), "mi\"rko");
EXPECT_EQ(literal->token_position_, 2);
}
TYPED_TEST(CypherMainVisitorTest, StringLiteralEscapedChars) {
@ -622,7 +628,8 @@ TYPED_TEST(CypherMainVisitorTest, StringLiteralEscapedChars) {
auto *literal = dynamic_cast<PrimitiveLiteral *>(
return_clause->body_.named_expressions[0]->expression_);
ASSERT_TRUE(literal);
ASSERT_EQ(literal->value_.Value<std::string>(), "\\'\"\b\b\f\f\n\n\r\r\t\t");
EXPECT_EQ(literal->value_.Value<std::string>(), "\\'\"\b\b\f\f\n\n\r\r\t\t");
EXPECT_EQ(literal->token_position_, 2);
}
TYPED_TEST(CypherMainVisitorTest, StringLiteralEscapedUtf16) {
@ -632,7 +639,8 @@ TYPED_TEST(CypherMainVisitorTest, StringLiteralEscapedUtf16) {
auto *literal = dynamic_cast<PrimitiveLiteral *>(
return_clause->body_.named_expressions[0]->expression_);
ASSERT_TRUE(literal);
ASSERT_EQ(literal->value_.Value<std::string>(), u8"\u221daaa\u221daaa");
EXPECT_EQ(literal->value_.Value<std::string>(), u8"\u221daaa\u221daaa");
EXPECT_EQ(literal->token_position_, 2);
}
TYPED_TEST(CypherMainVisitorTest, StringLiteralEscapedUtf32) {
@ -642,8 +650,9 @@ TYPED_TEST(CypherMainVisitorTest, StringLiteralEscapedUtf32) {
auto *literal = dynamic_cast<PrimitiveLiteral *>(
return_clause->body_.named_expressions[0]->expression_);
ASSERT_TRUE(literal);
ASSERT_EQ(literal->value_.Value<std::string>(),
EXPECT_EQ(literal->value_.Value<std::string>(),
u8"\U0001F600aaaa\U0001F600aaaaaaaa");
EXPECT_EQ(literal->token_position_, 2);
}
TYPED_TEST(CypherMainVisitorTest, DoubleLiteral) {
@ -653,7 +662,8 @@ TYPED_TEST(CypherMainVisitorTest, DoubleLiteral) {
auto *literal = dynamic_cast<PrimitiveLiteral *>(
return_clause->body_.named_expressions[0]->expression_);
ASSERT_TRUE(literal);
ASSERT_EQ(literal->value_.Value<double>(), 3.5);
EXPECT_EQ(literal->value_.Value<double>(), 3.5);
EXPECT_EQ(literal->token_position_, 2);
}
TYPED_TEST(CypherMainVisitorTest, DoubleLiteralExponent) {
@ -663,7 +673,8 @@ TYPED_TEST(CypherMainVisitorTest, DoubleLiteralExponent) {
auto *literal = dynamic_cast<PrimitiveLiteral *>(
return_clause->body_.named_expressions[0]->expression_);
ASSERT_TRUE(literal);
ASSERT_EQ(literal->value_.Value<double>(), 0.5);
EXPECT_EQ(literal->value_.Value<double>(), 0.5);
EXPECT_EQ(literal->token_position_, 2);
}
TYPED_TEST(CypherMainVisitorTest, ListLiteral) {

2
tests/unit/query_stripper.cpp
View File

@ -5,7 +5,7 @@
#include "gtest/gtest.h"
#include "query/stripped.hpp"
#include "query/frontend/stripped.hpp"
#include "query/typed_value.hpp"
using query::TypedValue;