memgraph demo first usable version

debug/log.hpp

@@ -14,7 +14,6 @@ class Log
 public:
     enum class Level : std::uint_fast8_t { Debug, Info, Warn, Error };
 
-private:
     struct Message
     {
         std::chrono::system_clock::time_point time;

demo/config/config.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+
+import json
+from util import get_env, set_modul_attrs
+
+host = "0.0.0.0"
+
+port = 5000
+
+log_level = 'INFO'
+
+try:
+    set_modul_attrs(__name__, json.loads(get_env('CONFIG')))
+except:
+    pass

demo/demo_server_init.py

@@ -7,24 +7,13 @@ via the MEMGRAPH_DEMO environtment variable. Available environments
 are: debug, prod.
 '''
 
-import os
 import logging
 
+from util import get_env
 from simulation.web_server import SimulationWebServer
 
-
-def fetch_env(env_name, default=None):
-    '''
-    Fetches environment variable.
-    '''
-    if env_name in os.environ:
-        return os.environ[env_name]
-
-    return default
-
-
-environment = fetch_env('MEMGRAPH_DEMO', 'debug')
-wsgi = fetch_env('MEMGRAPH_DEMO_WSGI', 'werkzeug')
+environment = get_env('MEMGRAPH_DEMO', 'debug')
+wsgi = get_env('MEMGRAPH_DEMO_WSGI', 'werkzeug')
 
 
 def _init():

demo/service_init.py

@@ -0,0 +1,7 @@
+# -*- coding: UTF-8 -*-
+
+#   TODO init object by module name
+
+# something like
+# module_name, class_name = module_class_string.rsplit(".", 1)
+# return getattr(importlib.import_module(module_name), class_name)()

demo/static/demo.html

@@ -96,7 +96,7 @@
     </div>
 
     <!-- the graph -->
-    <div class="col s8">
+    <div class="col s10">
       <div class="card">
         <div class="card-content">
           <div id="chart" style="height: 750px;">
@@ -107,6 +107,7 @@
     </div>
 
     <!-- memgraph cards -->
+    <!--
     <div class="col s2">
       <div id="q-1-0" class="card">
         <div class="qps valign-wrapper">
@@ -169,6 +170,7 @@
         <textarea id="query">MATCH (n:Person) RETURN n</textarea>
       </div>
     </div>
+    -->
   </div>
 
   <script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>

demo/static/demo.js

@@ -100,6 +100,7 @@
     }
   }
 
+  // put \n on module 2 space positions
   var put_new_line_mod_2 = function(array) {
     let join_array = array.map(function(o, i) {
       if (i % 2 == 0)
@@ -115,6 +116,13 @@
     });
   };
 
+  // --  variable part  ----
+  let running = false;
+  let value = 0;
+  let maxQps = 10000;
+  let cards = [];
+  let line = [];
+  let sec = 0;
   var queries = [
     "CREATE (n{id:@}) RETURN n",
     "MATCH (n{id:#}),(m{id:#}) CREATE (n)-[r:test]->(m) RETURN r",
@@ -122,47 +130,86 @@
     "MATCH (n{id:#}) RETURN n",
     "MATCH (n{id:#})-[r]->(m) RETURN count(r)"
   ];
+  var params = {
+    host: "localhost",
+    port: "7474",
+    connections: 16,
+    duration: 1,
+    queries: queries
+  };
+  // -----------------------
 
+  // server control functions
+  var start = function() {
+    $.ajax({url:'/start', type:"POST", success: function(data){}});
+  };
+  var stop = function() {
+    $.ajax({url:'/stop', type:"POST", success: function(data){}});
+  };
+  var registerParams = function() {
+    $.ajax({
+      url:'/params', type:"POST", data:JSON.stringify(params),
+      contentType:"application/json; charset=utf-8",
+      success: function(data){
+      }
+    });
+  };
+  registerParams();
+
+  // setup cards
+  queries.forEach(function(query, i) {
+    query = put_new_line_mod_2(query.split(" ")).join('');
+    cards.push(new QueryCard($('#q-0-' + i.toString())[0], maxQps));
+    cards[i].set_query(query);
+  });
+
+  // start stop button
   $("#running-button").click(function() {
     running = !running;
     if (running) {
       $(this).text('STOP');
-      run();
-      updateGraph();
+      start(); 
+      update();
     }
-    if (!running)
+    if (!running) {
       $(this).text('START');
+      stop();
+      // sec = 0;
+      // line = [];
+    }
   });
+ 
+  // update only line on the graph 
+  var updateGraph = function() {
+      let newData = [{
+        values: line,
+        key: 'QPS',
+        color: '#ff0000'
+      }];
+      chartData.datum(newData).transition().duration(500).call(chart);
+  }
 
-  // counters init
-  let running = false;
-  let value = 0;
-  let maxQps = 15000;
-
-  // cards init
-  let neo4jCards = [];
-  let memgraphCards = [];
-  queries.forEach(function(query, i) {
-    query = put_new_line_mod_2(query.split(" ")).join('');
-    neo4jCards.push(new QueryCard($('#q-0-' + i.toString())[0], maxQps));
-    neo4jCards[i].set_query(query);
-    memgraphCards.push(new QueryCard($('#q-1-' + i.toString())[0], maxQps));
-    memgraphCards[i].set_query(query);
-  });
-
-  // cards update
-  function run() {
-    if (!running)
+  // update
+  function update() {
+    if (!running) {
+      stop();
       return;
+    }
     setTimeout(() => {
-      value += 10;
-      if(value >= maxQps)
-        value = 0;
-      queries.forEach(function(query, i) {
-        neo4jCards[i].set_value(Math.round(1000 + Math.random() * 3000));
-        memgraphCards[i].set_value(Math.round(7000 + Math.random() * 7000));
+      $.ajax({
+        url:'/stats', type:"GET",
+        success: function(data){
+          if (!data || !data.total || !data.per_query)
+            return
+          sec = sec + 1;
+          line.push({x: sec, y: data.total});
+          data.per_query.forEach(function(speed, i) {
+            cards[i].set_value(Math.round(speed));
+          });
+          updateGraph();
+        }
       });
-      run();
+      update();
     }, 1000);
   }
 
@@ -194,33 +241,4 @@
     return chart;
   });
 
-  // graph update
-  let x = 0;
-  function updateGraph() {
-    if (!running)
-      return;
-    setTimeout(() => {
-      x += 1;
-      if (x > 100)
-        x = 0
-      var memgraphLine = [];
-      var neo4jLine = [];
-      for (var i = 0; i < x; i++) {
-        memgraphLine.push({x: i, y: 100 * Math.random() + 1000});
-        neo4jLine.push({x: i, y: 100 * Math.random() + 50});
-      }
-      var newData = [{
-        values: memgraphLine,
-        key: 'Memgraph',
-        color: '#ff0000'
-      }, {
-        values: neo4jLine,
-        key: 'Neo4j',
-        color: '#0000ff'
-      }];
-      chartData.datum(newData).transition().duration(500).call(chart);
-      updateGraph();
-    }, 1000);
-  }
-
 })();

demo/util.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+
+'''
+python utilities
+'''
+
+import os
+import sys
+import json
+
+
+def get_env(env_name, default=None, type=None):
+    '''
+    Fetches environment variable.
+    '''
+    if env_name in os.environ and type:
+        return type(os.environ[env_name])
+    elif env_name in os.environ:
+        return os.environ[env_name]
+    elif type:
+        return type(default)
+
+    return default
+
+
+def set_modul_attrs(name, values):
+    """
+    Updates the module with the values.
+
+    :param name: str, module name, if this method call is located
+                 inside module then use __name__
+    :param values: dict, contains new key values.
+    """
+    #   get the config module (this module)
+    config_module = sys.modules[name]
+
+    #   set all the values for given keys on this module
+    for key, value in values.items():
+        setattr(config_module, key, value)
+
+
+def load_module_attrs(name, env_flag):
+    """
+    A function for initializing module from config files (JSON formatted).
+    Called when loading the module. The module is specified by it's name.
+
+    :param name: str, module name, if this method call is located
+                 inside module then use __name__
+    :param env_flag: str, name of the env variable in which is path to JSON
+                     file from which module attributes are going to be
+                     populated
+    :param argv_flag: str, name of program argument which will define path
+                      to the json file
+    """
+    config_path = get_env(env_flag)
+    if config_path:
+        with open(config_path, 'r') as config_file:
+            set_modul_attrs(name, json.load(config_file))

demo/web_service.py

@@ -3,9 +3,9 @@
 from flask import Flask
 
 
-class WebService(Flask):
+class WebService(object):
 
-    def __init__(self):
+    def __init__(self, name):
         '''
         '''
         self.server = Flask(__name__)

demo/worker/benchmark.pyx

@@ -3,7 +3,7 @@
 from libcpp.string cimport string
 from libcpp.vector cimport vector
 
-cdef extern from "benchmark.hpp":
+cdef extern from "benchmark.hpp" nogil:
     string benchmark_json(const string& host,
                           const string& port,
                           int connections_per_query,
@@ -14,6 +14,7 @@ cdef extern from "benchmark.hpp":
 def benchmark(host, port, connections_per_query, duration, queries):
     '''
     '''
-    return benchmark_json(<const string&> host, <const string&> port,
-                          <int> connections_per_query, <double> duration,
-                          <const vector[string]&> queries)
+    result = benchmark_json(<const string&> host, <const string&> port,
+                            <int> connections_per_query, <double> duration,
+                            <const vector[string]&> queries)
+    return result

demo/worker/client.py

@@ -3,7 +3,8 @@
 from os import path
 from json import loads
 from subprocess import check_output
-from benchmark import benchmark
+
+from .benchmark import benchmark
 
 
 def subprocess_client(args):
@@ -46,7 +47,7 @@ def main_wrapped():
     '''
     Example of cython call.
     '''
-    args = [b"localhost", b"7474", 16, 1,
+    args = [b"localhost", b"7474", 1, 1,
             [b"CREATE (n{id:@}) RETURN n", b"CREATE (n{id:@}) RETURN n"]]
     result = wrapped_client(*args)
     return result

demo/worker/setup_runner.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 python setup.py build_ext --inplace
-mv benchmark.cpp __benchmark__.cpp
+# mv benchmark.cpp __benchmark__.cpp

demo/worker/worker_web_service.py

@@ -2,15 +2,17 @@
 
 import logging
 import threading
+from os import path
 from flask import request, jsonify
 
-from .. import WebService
-from . import wrapped_client
+from web_service import WebService
+# from .client import wrapped_client
+from .client import subprocess_client
 
 log = logging.getLogger(__name__)
 
 
-class WorkerWebServer(WebService):
+class WorkerWebService(WebService):
     '''
     Memgraph worker web server. For now it wraps the flask server.
     '''
@@ -19,8 +21,8 @@ class WorkerWebServer(WebService):
         '''
         Instantiates the flask web server.
         '''
-        super().__init__()
-        self.params_data = None
+        super().__init__(__name__)
+        self.params_data = {}
         self.is_simulation_running = False
         self.stats_data = None
         self.setup_routes()
@@ -29,13 +31,28 @@ class WorkerWebServer(WebService):
         '''
         Setup all routes.
         '''
-        super().__init__()
+        super().setup_routes()
+        self.add_route('/', self.index, 'GET')
+        self.add_route('/<path:path>', self.static, 'GET')
         self.add_route('/start', self.start, 'POST')
         self.add_route('/stop', self.stop, 'POST')
         self.add_route('/stats', self.stats, 'GET')
         self.add_route('/params', self.params_get, 'GET')
         self.add_route('/params', self.params_set, 'POST')
 
+    def index(self):
+        '''
+        Serves demo.html on the index path.
+        '''
+        print('index')
+        return self.server.send_static_file('demo.html')
+
+    def static(self, path):
+        '''
+        Serves other static files.
+        '''
+        return self.server.send_static_file(path)
+
     def run_simulation(self):
         '''
         If flag is_simulation_running flag is up (True) the executor
@@ -45,10 +62,31 @@ class WorkerWebServer(WebService):
         log.info('new simulation run')
 
         while self.is_simulation_running:
-            self.stats_data = wrapped_client(*self.params_data)
+            #   cython call TODO relase the GIL
+            # params = [
+            #     self.params_data['host'].encode('utf-8'),
+            #     self.params_data['port'].encode('utf-8'),
+            #     self.params_data['connections'],
+            #     self.params_data['duration'],
+            #     list(map(lambda x: x.encode('utf-8'),
+            #              self.params_data['queries']))
+            # ]
+            # data = wrapped_client(*params)
+
+            #   subprocess call
+            params = [
+                str(self.params_data['host']),
+                str(self.params_data['port']),
+                str(self.params_data['connections']),
+                str(self.params_data['duration'])
+            ] + list(map(lambda x: str(x), self.params_data['queries']))
+            exe = path.join(path.dirname(path.abspath(__file__)),
+                            "benchmark_json.out")
+            self.stats_data = subprocess_client([exe] + params)
 
     def start(self):
         '''
+        Starts run in a separate thread.
         '''
         self.is_simulation_running = True
         t = threading.Thread(target=self.run_simulation, daemon=True)
@@ -57,13 +95,14 @@ class WorkerWebServer(WebService):
 
     def stop(self):
         '''
+        Stops the worker run.
         '''
         self.is_simulation_running = False
         return ('', 204)
 
     def stats(self):
         '''
-        Returns the simulation stats. Queries per second.
+        Returns the worker stats. Queries per second.
         '''
         if not self.stats_data:
             return ('', 204)
@@ -72,13 +111,13 @@ class WorkerWebServer(WebService):
 
     def params_get(self):
         '''
-        Returns simulation parameters.
+        Returns worker parameters.
         '''
-        return jsonify(self.simulation_params.json_data())
+        return jsonify(self.params_data)
 
     def params_set(self):
         '''
-        Sets simulation parameters.
+        Sets worker parameters.
         '''
         data = request.get_json()
 
@@ -86,6 +125,6 @@ class WorkerWebServer(WebService):
 
         for param in param_names:
             if param in data:
-                setattr(self.params_data, param, data[param])
+                self.params_data[param] = data[param]
 
         return self.params_get()

demo/worker_web_service_init.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+
+import logging
+
+from config import config
+from worker.worker_web_service import WorkerWebService
+
+
+def _init():
+    '''
+    Defines log level.
+    '''
+    logging.basicConfig(level=config.log_level)
+
+    return WorkerWebService().server
+
+app = _init()
+
+if __name__ == '__main__':
+    app.run(host=config.host, port=config.port)