query stripping and stripped string hash calculation

cypher/compiler.hpp

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "cypher_lexer.hpp"
+#include "tokenizer/cypher_lexer.hpp"
 #include "parser.hpp"
 
 namespace cypher

cypher/parser.hpp

@@ -2,9 +2,8 @@
 
 #include "cypher.h"
 #include "token.hpp"
-
-#include "cypher_lexer.hpp"
 #include "ast/tree.hpp"
+#include "tokenizer/cypher_lexer.hpp"
 
 void* cypher_parserAlloc(void* (*allocProc)(size_t));
 void  cypher_parser(void*, int, Token*, ast::Ast* ast);

cypher/token.hpp

@@ -10,6 +10,26 @@ struct Token
     unsigned long id;
     std::string value;
 
+    /*
+     * Token is "True" if it's id is bigger than zero. Because
+     * lexer ids are all bigger than zero.
+     *
+     * This object could be used in while loop as a condition. 
+     * E.g.:
+     * while (auto token = ...)
+     * {
+     * }
+     */
+    operator bool() const
+    {
+        return id > 0;
+    }
+
+    /*
+     * Ostream operator
+     *
+     * Prints token id and value in single line.
+     */
     friend std::ostream& operator<<(std::ostream& stream, const Token& token)
     {
         return stream << "TOKEN id = " << token.id

cypher/tokenizer/cypher_lexer.hpp

@@ -1,7 +1,6 @@
 #pragma once
 
-#include "cypher.h"
-
+#include "cypher/cypher.h"
 #include "lexer.hpp"
 
 class CypherLexer : public Lexer

cypher/tokenizer/lexer.hpp

@@ -7,12 +7,12 @@
 // auto_ptr<lexertl::detail::basic_re_token<char, char> > is deprecated
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#include "lexertl/lexertl/generator.hpp"
-#include "lexertl/lexertl/lookup.hpp"
+#include "cypher/lexertl/lexertl/generator.hpp"
+#include "cypher/lexertl/lexertl/lookup.hpp"
 #pragma GCC diagnostic pop
 
-#include "errors.hpp"
-#include "token.hpp"
+#include "cypher/errors.hpp"
+#include "cypher/token.hpp"
 
 class Lexer
 {

query_engine/compile.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 # TODO: create Makefile or cmake script
-clang++ -std=c++1y -I../ main.cpp ../cypher/cypher.cpp -o engine
+clang++ -std=c++1y -g -I../ main.cpp ../cypher/cypher.cpp -o engine

query_engine/query_engine.hpp

@@ -2,11 +2,13 @@
 
 #include <iostream>
 
+#include "query_stripper.hpp"
 #include "query_traverser.hpp"
 #include "code_generator.hpp"
 #include "code_compiler.hpp"
 #include "query_executor.hpp"
 #include "query_result.hpp"
+#include "utils/hashing/fnv.hpp"
 
 using std::cout;
 using std::endl;
@@ -21,13 +23,19 @@ class QueryEngine
 public:
     QueryResult execute(const std::string& query)
     {
-        traverser.build_tree(query);
-        traverser.traverse();
+        auto stripped = stripper.strip(query);
+        cout << "STRIPPED: " << stripped << endl;
+        auto stripped_hash = fnv(stripped);
+        cout << "STRIPPED HASH: " << stripped_hash << endl;
+
+        // traverser.build_tree(query);
+        // traverser.traverse();
         return QueryResult();
     }
 
 private:
     // TODO: use IoC or something similar
+    QueryStripper stripper;
     QueryTraverser traverser;
     CodeGenerator generator;
     CodeCompiler compiler;

query_engine/query_stripper.hpp

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <string>
+#include <regex>
+
+#include "cypher/tokenizer/cypher_lexer.hpp"
+#include "cypher/cypher.h"
+
+class QueryStripper
+{
+public:
+
+    // TODO: extract parameters
+
+    std::string strip(const std::string& query)
+    {
+        auto tokenizer = lexer.tokenize(query);
+        std::string stripped = "";
+        int counter = 0;
+        while (auto token = tokenizer.lookup())
+        {
+            // TODO: do this more generic via template metaprogramming
+            if (token.id == TK_STR || token.id == TK_INT ||
+                token.id == TK_FLOAT) {
+                stripped += "@" + std::to_string(counter++);
+            } else {
+                stripped += token.value;
+            }
+        }
+        return stripped;
+    }
+
+private:
+    CypherLexer lexer;
+};