From 35482711a89399a41bffb492b46c56cbceea31e4 Mon Sep 17 00:00:00 2001
From: Marko Budiselic <mbudiselicbuda@gmail.com>
Date: Sat, 5 Mar 2016 10:10:57 +0100
Subject: [PATCH] demo substitutor implementation
---
api/resources/ping.hpp | 15 ++++
api/response_json.hpp | 6 +-
api/restful/restful_resource.hpp | 2 +-
demo/demo_server.docker | 12 +--
demo/demo_server_init.py | 53 +++++++++++++
demo/demo_server_run.py | 31 --------
demo/memgraph_demo.build | 5 --
demo/memgraph_demo.docker | 8 --
demo/memgraph_server.docker | 10 +++
demo/neo4j_demo.run | 5 --
demo/requirements.txt | 18 +++++
...{memgraph_demo.run => run_memgraph_server} | 0
demo/run_neo4j_server | 5 ++
demo/simulation/executor.py | 21 +++--
demo/simulation/task.py | 9 +++
demo/simulation/web_server.py | 70 +++++++++--------
demo/substitutor.py | 77 +++++++++++++++++++
17 files changed, 251 insertions(+), 96 deletions(-)
create mode 100644 api/resources/ping.hpp
create mode 100644 demo/demo_server_init.py
delete mode 100644 demo/demo_server_run.py
delete mode 100755 demo/memgraph_demo.build
delete mode 100644 demo/memgraph_demo.docker
create mode 100644 demo/memgraph_server.docker
delete mode 100755 demo/neo4j_demo.run
create mode 100644 demo/requirements.txt
rename demo/{memgraph_demo.run => run_memgraph_server} (100%)
create mode 100755 demo/run_neo4j_server
create mode 100644 demo/substitutor.py
diff --git a/api/resources/ping.hpp b/api/resources/ping.hpp
new file mode 100644
index 000000000..ff993dda4
--- /dev/null
+++ b/api/resources/ping.hpp
@@ -0,0 +1,15 @@
+#pragma once
+
+#include "api/restful/resource.hpp"
+
+#pragma url /ping
+class Ping : public Resource<Ping, GET>
+{
+public:
+ using Resource::Resource;
+
+ void get(sp::Request& req, sp::Response& res)
+ {
+ return res.send(http::Status::NoContent, "");
+ }
+};
diff --git a/api/response_json.hpp b/api/response_json.hpp
index 8a0a26231..5c659e271 100644
--- a/api/response_json.hpp
+++ b/api/response_json.hpp
@@ -45,9 +45,9 @@ std::string vertex_create_response(const Vertex::Accessor& vertex_accessor)
writer->String("data");
writer->StartObject();
- RapidJsonStringWriter dataBuffer(writer);
- auto properties = vertex_accessor.properties();
- properties.accept(dataBuffer);
+ // RapidJsonStringWriter dataBuffer(writer);
+ // auto properties = vertex_accessor.properties();
+ // properties.accept(dataBuffer);
writer->EndObject();
writer->EndObject();
diff --git a/api/restful/restful_resource.hpp b/api/restful/restful_resource.hpp
index 72eed6639..8c7482bbd 100644
--- a/api/restful/restful_resource.hpp
+++ b/api/restful/restful_resource.hpp
@@ -5,7 +5,7 @@
#include "speedy/speedy.hpp"
#include "utils/crtp.hpp"
-#include "storage/model/properties/jsonwriter.hpp"
+#include "storage/model/properties/traversers/jsonwriter.hpp"
/** @brief GET handler method for the resource
* Contains the code for registering GET handler for a URL to Speedy
diff --git a/demo/demo_server.docker b/demo/demo_server.docker
index 38df666d5..b0bca606f 100644
--- a/demo/demo_server.docker
+++ b/demo/demo_server.docker
@@ -1,15 +1,17 @@
FROM ubuntu:16.04
RUN apt-get update
-RUN apt-get install -y python3.5 python3-pip
-RUN apt-get install -y python3-setuptools
+RUN apt-get install -y python3.5 python3.5-dev python3-pip python3-setuptools
# RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-RUN pip3 install flask requests
+RUN pip3 install flask
COPY simulation /app/simulation
-COPY demo_server_run.py /app/demo_server_run.py
+COPY demo_server_run.py /app/demo_server_init.py
WORKDIR /app
-CMD ["python3", "demo_server_run.py"]
+ENV MEMGRAPH_DEMO prod
+
+# uwsgi --http-socket 0.0.0.0:8080 --module demo_server_init:app --master --enable-threads
+CMD ["python3", "demo_server_init.py"]
diff --git a/demo/demo_server_init.py b/demo/demo_server_init.py
new file mode 100644
index 000000000..6f1932824
--- /dev/null
+++ b/demo/demo_server_init.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+The demo server init script. Environment could be configured
+via the MEMGRAPH_DEMO environtment variable. Available environments
+are: debug, prod.
+'''
+
+import os
+import logging
+
+from simulation.web_server import SimulationWebServer
+
+
+def fetch_env(env_name, default=None):
+ '''
+ Fetches environment variable.
+ '''
+ if env_name in os.environ:
+ return os.environ[env_name]
+
+ return default
+
+
+environment = fetch_env('MEMGRAPH_DEMO', 'debug')
+wsgi = fetch_env('MEMGRAPH_DEMO_WSGI', 'werkzeug')
+
+
+def _init():
+ '''
+ Initialzies logging level and server.
+ '''
+ if environment == 'prod':
+ logging.basicConfig(level=logging.WARNING)
+ elif environment == 'test':
+ logging.basicConfig(level=logging.INFO)
+ else:
+ logging.basicConfig(level=logging.DEBUG)
+
+ return SimulationWebServer().server
+
+
+app = _init()
+
+
+if __name__ == '__main__':
+ if wsgi == 'gevent':
+ from gevent.wsgi import WSGIServer
+ http_server = WSGIServer(('', 8080), app)
+ http_server.serve_forever()
+ else:
+ app.run(host="0.0.0.0", port=8080)
diff --git a/demo/demo_server_run.py b/demo/demo_server_run.py
deleted file mode 100644
index ff628aced..000000000
--- a/demo/demo_server_run.py
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import os
-import logging
-
-from simulation.web_server import SimulationWebServer
-
-
-def main():
- '''
- The demo server run script. Environment could be configured
- via the MEMGRAPH_DEMO environtment variable. Available environments
- are: debug, prod.
- '''
- if 'MEMGRAPH_DEMO' in os.environ:
- environment = os.environ['MEMGRAPH_DEMO']
- else:
- environment = "debug"
-
- frontend_server = SimulationWebServer()
-
- if environment == 'prod':
- logging.basicConfig(level=logging.WARNING)
- frontend_server.run("0.0.0.0", 8080, False)
- elif environment == 'debug':
- logging.basicConfig(level=logging.INFO)
- frontend_server.run("0.0.0.0", 8080, True)
-
-if __name__ == '__main__':
- main()
diff --git a/demo/memgraph_demo.build b/demo/memgraph_demo.build
deleted file mode 100755
index 5462c4406..000000000
--- a/demo/memgraph_demo.build
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-cd ..
-docker rmi memgraph_demo
-docker build -t memgraph_demo -f demo/memgraph_demo.docker .
diff --git a/demo/memgraph_demo.docker b/demo/memgraph_demo.docker
deleted file mode 100644
index dfd379cd7..000000000
--- a/demo/memgraph_demo.docker
+++ /dev/null
@@ -1,8 +0,0 @@
-FROM ubuntu:16.04
-
-RUN apt-get update
-RUN apt-get install -y git
-
-RUN git clone https://pullbot:JnSdamFGKOanF1@phabricator.tomicevic.com/diffusion/MG/memgraph.git
-
-CMD ["/bin/bash"]
diff --git a/demo/memgraph_server.docker b/demo/memgraph_server.docker
new file mode 100644
index 000000000..f4614ab62
--- /dev/null
+++ b/demo/memgraph_server.docker
@@ -0,0 +1,10 @@
+FROM ubuntu:16.04
+
+RUN apt-get update
+RUN apt-get install -y git
+
+RUN mkdir /app
+WORKDIR /app
+RUN git clone --recursive https://${MEMGRAPH_PULL_USER}:${MEMGRAPH_PULL_PASS}@phabricator.tomicevic.com/diffusion/MG/memgraph.git
+
+CMD ["/bin/bash"]
diff --git a/demo/neo4j_demo.run b/demo/neo4j_demo.run
deleted file mode 100755
index c55b522fe..000000000
--- a/demo/neo4j_demo.run
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-docker stop neo4j_demo
-docker rm neo4j_demo
-docker run -d --name neo4j_demo --net=host -p 7474:7474 neo4j
diff --git a/demo/requirements.txt b/demo/requirements.txt
new file mode 100644
index 000000000..5652c67de
--- /dev/null
+++ b/demo/requirements.txt
@@ -0,0 +1,18 @@
+Flask==0.10.1
+Jinja2==2.8
+MarkupSafe==0.23
+Werkzeug==0.11.4
+decorator==4.0.9
+gevent==1.1rc5
+greenlet==0.4.9
+ipython==4.1.1
+ipython-genutils==0.1.0
+itsdangerous==0.24
+path.py==8.1.2
+pexpect==4.0.1
+pickleshare==0.6
+ptyprocess==0.5.1
+requests==2.9.1
+simplegeneric==0.8.1
+traitlets==4.1.0
+uWSGI==2.0.12
diff --git a/demo/memgraph_demo.run b/demo/run_memgraph_server
similarity index 100%
rename from demo/memgraph_demo.run
rename to demo/run_memgraph_server
diff --git a/demo/run_neo4j_server b/demo/run_neo4j_server
new file mode 100755
index 000000000..37fce1bdf
--- /dev/null
+++ b/demo/run_neo4j_server
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+docker stop neo4j_server
+docker rm neo4j_server
+docker run -d --name neo4j_server --net=host -p 7474:7474 neo4j
diff --git a/demo/simulation/executor.py b/demo/simulation/executor.py
index 8d415fc70..ab7717abe 100644
--- a/demo/simulation/executor.py
+++ b/demo/simulation/executor.py
@@ -3,7 +3,7 @@
import time
import logging
import itertools
-import requests
+import urllib.request
from concurrent.futures import ProcessPoolExecutor
from .epoch_result import SimulationEpochResult
@@ -61,13 +61,16 @@ class SimulationExecutor(object):
:param query: str, query string
'''
- requests.post('http://localhost:7474/db/data/transaction/commit',
- json={
- "statements": [
- {"statement": query}
- ]
- },
- headers={'Authorization': 'Basic bmVvNGo6cGFzcw=='})
+ # requests.post('http://localhost:7474/db/data/transaction/commit',
+ # json={
+ # "statements": [
+ # {"statement": query}
+ # ]
+ # },
+ # headers={'Authorization': 'Basic bmVvNGo6cGFzcw=='})
+ # requests.get('http://localhost:7474/db/data/ping')
+ response = urllib.request.urlopen('http://localhost:7474/db/data/ping')
+ response.read()
def iteration(self, task):
'''
@@ -108,6 +111,8 @@ class SimulationExecutor(object):
with self.pool() as executor:
+ log.info('pool iter')
+
# execute all tasks
futures = [executor.submit(self.iteration, task)
for task in self.params.tasks
diff --git a/demo/simulation/task.py b/demo/simulation/task.py
index bdb9bc3e8..0c8f267c7 100644
--- a/demo/simulation/task.py
+++ b/demo/simulation/task.py
@@ -13,3 +13,12 @@ class SimulationTask(object):
'''
self.id = id
self.query = query
+
+ def json_data(self):
+ '''
+ :returns: dict with all elements
+ '''
+ return {
+ "id": self.id,
+ "query": self.query
+ }
diff --git a/demo/simulation/web_server.py b/demo/simulation/web_server.py
index 083833b58..08e2bbe1b 100644
--- a/demo/simulation/web_server.py
+++ b/demo/simulation/web_server.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
+import json
import logging
import threading
from flask import Flask, request, jsonify
@@ -11,7 +12,7 @@ from .task import SimulationTask
log = logging.getLogger(__name__)
-class SimulationWebServer():
+class SimulationWebServer(object):
'''
Memgraph demo fontend server. For now it wraps the flask server.
'''
@@ -20,44 +21,45 @@ class SimulationWebServer():
'''
Instantiates the flask web server.
'''
- self.server = Flask(__name__)
self.is_simulation_running = False
self.simulation_stats = None
self.simulation_params = SimulationParams()
- self.simulation_executor = \
- SimulationExecutor().setup(self.simulation_params)
+ self.simulation_executor = SimulationExecutor()
+ self.server = Flask(__name__)
+ self.setup_routes()
+ self.server.before_first_request(self.before_first_request)
def setup_routes(self):
'''
- Setup all available rutes:
- GET /ping
- POST /tasks
- POST /start
- POST /stop
- GET /stats
- GET /params
- POST /params
+ Setup all routes.
'''
- self.server.add_url_rule('/ping', 'ping', self.ping)
- self.server.add_url_rule('/tasks', 'tasks', self.tasks,
- methods=['POST'])
- self.server.add_url_rule('/start', 'start', self.start,
- methods=['POST'])
- self.server.add_url_rule('/stop', 'stop', self.stop,
- methods=['POST'])
- self.server.add_url_rule('/stats', 'stats', self.stats,
- methods=['GET'])
- self.server.add_url_rule('/params', 'params_get', self.params_get,
- methods=['GET'])
- self.server.add_url_rule('/params', 'params_set', self.params_set,
- methods=['POST'])
+ self.add_route('/ping', self.ping, 'GET')
+ self.add_route('/tasks', self.tasks_get, 'GET')
+ self.add_route('/tasks', self.tasks_set, 'POST')
+ self.add_route('/start', self.start, 'POST')
+ self.add_route('/stop', self.stop, 'POST')
+ self.add_route('/stats', self.stats, 'GET')
+ self.add_route('/params', self.params_get, 'GET')
+ self.add_route('/params', self.params_set, 'POST')
- def run(self, host="127.0.0.1", port=8080, debug=False):
+ def before_first_request(self):
'''
- Runs the server. Before run, routes are initialized.
+ Initializes simulation executor before first request.
'''
- self.setup_routes()
- self.server.run(host=host, port=port, debug=debug)
+ log.info('before first request')
+ self.simulation_executor.setup(self.simulation_params)
+
+ def add_route(self, route, code_method, http_method):
+ '''
+ Registers URL rule
+
+ :param route: str, route string
+ :param object_method: object method responsible for the
+ request handling
+ :param http_method: name of http method
+ '''
+ self.server.add_url_rule(route, '%s_%s' % (route, http_method),
+ code_method, methods=[http_method])
def ping(self):
'''
@@ -65,7 +67,15 @@ class SimulationWebServer():
'''
return ('', 204)
- def tasks(self):
+ def tasks_get(self):
+ '''
+ Retutns all defined tasks.
+ '''
+ return json.dumps(
+ [task.json_data() for task in self.simulation_params.tasks]
+ )
+
+ def tasks_set(self):
'''
Register tasks. Task is object that encapsulates single query data.
'''
diff --git a/demo/substitutor.py b/demo/substitutor.py
new file mode 100644
index 000000000..7125fb0a5
--- /dev/null
+++ b/demo/substitutor.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+
+import sys
+import random
+import string
+
+
+char_options = string.ascii_lowercase + string.digits
+
+
+def random_integer(start=0, end=sys.maxsize):
+ '''
+ :returns: random integer between start and end
+ '''
+ return random.randint(start, end)
+
+
+def random_float(start=0, end=1):
+ '''
+ :returns: random float between start and end (uniform distribution)
+ '''
+ return random.uniform(start, end)
+
+
+def random_bool():
+ '''
+ :returns: random bool value
+ '''
+ return bool(random.getrandbits(1))
+
+
+def random_string(size=5):
+ '''
+ :param size: int, string size
+
+ :returns: random string of specific size build from ascii_lowercase chars
+ and digits
+ '''
+ return ''.join([random.choice(char_options)
+ for _ in range(size)])
+
+
+placeholders = {
+ '#': random_integer,
+ '@': random_float,
+ '*': random_bool,
+ '^': random_string
+}
+
+
+def substitute(text='', placeholders=placeholders):
+ '''
+ Substitutes chars in text with values generated from functions placed
+ in placeholders dict.
+
+ :param text: str, substitutable text
+ :param placeholders: dict, key is char that will be substituted, value
+ is function that is going to be used to generate
+ a new value
+ '''
+ return ''.join((c if c not in placeholders else str(placeholders[c]())
+ for c in iter(text)))
+
+
+if __name__ == '__main__':
+
+ def test_1():
+ print([f() for f in [random_integer, random_float,
+ random_bool, random_string]])
+
+ def test_2():
+ return substitute('int # float @ bool * string ^')
+
+ def test_3():
+ print(test_2())
+
+ test_3()