github-repos/wget
1
0
Fork 0
mirror of https://github.com/mirror/wget.git synced 2025-03-22 07:40:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Shift to new Threading Model from Multiprocessing model

Browse Source 
This eliminated the use of Global Variables from the HTTPServer module
and uses the more correct threading.Threads module to spawn a new
server. However, on multi-core systems, the performance may deteriorate
when too many servers are launched due to CPython implementation of GIL.

Many artefacts from the old model still remain, sometimes as commenst,
else as executable code. These will be cleaned up soon
This commit is contained in:
Darshit Shah 2013-09-07 11:15:14 +05:30
parent e74c2ec25e
commit 886ac1a89b
3 changed files with 77 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
testenv/ChangeLog
View File

@ -1,3 +1,31 @@
2013-09-07 Darshit Shah <darnir@gmail.com>
* HTTPServer.py (StoppableHTTPServer.server_conf): Change global
variable fileSys to an object variable. This is good programming
practice and required for parallel-wget support.
(StoppableHTTPServer.server_forever): Edit overridden method to remove
the global queue variable. No longer required under the new working
(WgetHTTPRequestHandler.do_QUIT): Don't push fileSys through the queue
(_Handler): Rename class __Handler to _Handler to match Python's
encapsulation rules
(_Handler.do_POST): fileSys is now an object variable of the server
(_Handler.do_PUT): Same
(_Handler.send_put): Same
(_Handler.send_head): Same
(HTTPd): New class that wraps around the server for Threading
(create_server): Make new object of HTTPd.
(spawn_server): Start the thread created through create_server
(ret_fileSys): Removed method. No longer required.
* WgetTest.py (HTTPTest.__init__): Don't explicitly set
self.act_retcode. Instead toggle tests_passed boolean to set the
correct return code.
(HTTPTest.HTTP_setup): We no longer call HTTPServer.spawn_server to
start a new instance of the server.
(HTTPTest.init_HTTP_server): We no longer call the old
create_server(), spawn_server() methods. Instead use the new HTTPd
class interface to create new instances of the server
(HTTPTest.stop_HTTP_server): Don't ask server to return fileSys.
2013-09-07 Darshit Shah <darnir@gmail.com>
* Test-Post.py: Test basic functionality for sending HTTP POST

65
testenv/HTTPServer.py
View File

@ -3,6 +3,7 @@ from http.server import HTTPServer, BaseHTTPRequestHandler
from base64 import b64encode
from random import random
from hashlib import md5
import threading
import os
import re
@ -32,17 +33,22 @@ class StoppableHTTPServer (HTTPServer):
"""
self.server_configs = conf_dict
global fileSys
fileSys = filelist
self.fileSys = filelist
def serve_forever (self, q):
""" Override method allowing for programmatical shutdown process. """
def serve_forever (self):
self.stop = False
while not self.stop:
self.handle_request ()
""" def serve_forever (self, q):
# Override method allowing for programmatical shutdown process.
global queue
queue = q
self.stop = False
while not self.stop:
self.handle_request ()
"""
class WgetHTTPRequestHandler (BaseHTTPRequestHandler):
@ -76,13 +82,13 @@ class WgetHTTPRequestHandler (BaseHTTPRequestHandler):
return r_list
def do_QUIT (self):
queue.put (fileSys)
#queue.put (self.server.fileSys)
self.send_response (200)
self.end_headers ()
self.server.stop = True
class __Handler (WgetHTTPRequestHandler):
class _Handler (WgetHTTPRequestHandler):
""" Define Handler Methods for different Requests. """
@ -107,13 +113,13 @@ class __Handler (WgetHTTPRequestHandler):
self.rules = self.server.server_configs.get (path)
if not self.custom_response ():
return (None, None)
if path in fileSys:
if path in self.server.fileSys:
body_data = self.get_body_data ()
self.send_response (200)
self.send_header ("Content-type", "text/plain")
content = fileSys.pop (path) + "\n" + body_data
content = self.server.fileSys.pop (path) + "\n" + body_data
total_length = len (content)
fileSys[path] = content
self.server.fileSys[path] = content
self.send_header ("Content-Length", total_length)
self.finish_headers ()
try:
@ -128,7 +134,7 @@ class __Handler (WgetHTTPRequestHandler):
self.rules = self.server.server_configs.get (path)
if not self.custom_response ():
return (None, None)
fileSys.pop (path, None)
self.server.fileSys.pop (path, None)
self.send_put (path)
""" End of HTTP Request Method Handlers. """
@ -156,7 +162,7 @@ class __Handler (WgetHTTPRequestHandler):
def send_put (self, path):
body_data = self.get_body_data ()
self.send_response (201)
fileSys[path] = body_data
self.server.fileSys[path] = body_data
self.send_header ("Content-type", "text/plain")
self.send_header ("Content-Length", len (body_data))
self.finish_headers ()
@ -374,7 +380,7 @@ class __Handler (WgetHTTPRequestHandler):
"""
path = self.path[1:]
if path in fileSys:
if path in self.server.fileSys:
self.rules = self.server.server_configs.get (path)
for rule_name in self.rules:
@ -397,7 +403,7 @@ class __Handler (WgetHTTPRequestHandler):
else:
return (None, None)
content = fileSys.get (path)
content = self.server.fileSys.get (path)
content_length = len (content)
try:
self.range_begin = self.parse_range_header (
@ -428,20 +434,33 @@ class __Handler (WgetHTTPRequestHandler):
self.send_error (404, "Not Found")
return (None, None)
class HTTPd (threading.Thread):
server_class = StoppableHTTPServer
handler = _Handler
def __init__ (self, addr=None):
threading.Thread.__init__ (self)
if addr is None:
addr = ('localhost', 0)
self.server = self.server_class (addr, self.handler)
self.server_address = self.server.socket.getsockname()[:2]
def run (self):
self.server.serve_forever ()
def server_conf (self, file_list, server_rules):
self.server.server_conf (file_list, server_rules)
def create_server ():
server = StoppableHTTPServer (("localhost", 0), __Handler)
server = HTTPd ()
return server
def spawn_server (server):
global q
q = Queue()
server_process = Process (target=server.serve_forever, args=(q,))
server_process.start ()
def ret_fileSys ():
return (q.get (True))
#global q
#q = Queue()
#server_process = Process (target=server.serve_forever, args=(q,))
#server_process.start ()
server.start ()
# vim: set ts=8 sts=4 sw=3 tw=0 et :

15
testenv/WgetTest.py
View File

@ -204,13 +204,12 @@ class HTTPTest (CommonMethods):
self.HTTP_setup (name, pre_hook, test_params, post_hook)
except TestFailed as tf:
printer ("RED", "Error: " + tf.error)
self.act_retcode = 100
self.tests_passed = False
except Exception as ae:
printer ("RED", "Unhandled Exception Caught.")
print ( ae.__str__ ())
traceback.print_exc ()
self.act_retcode = 100
self.tests_passed = False
else:
printer ("GREEN", "Test Passed")
finally:
@ -239,7 +238,7 @@ class HTTPTest (CommonMethods):
getattr (self, test_func) (test_params[test_func])
HTTPServer.spawn_server (self.server)
#HTTPServer.spawn_server (self.server)
self.act_retcode = self.exec_wget (self.options, self.urls, self.domain)
self.stop_HTTP_Server ()
@ -251,16 +250,16 @@ class HTTPTest (CommonMethods):
getattr (self, post_hook_func) (post_hook[post_hook_func])
def init_HTTP_Server (self):
server = HTTPServer.create_server ()
return server
#server = HTTPServer.HTTPd ()
#server.start (self)
#server = HTTPServer.create_server ()
#return server
server = HTTPServer.HTTPd ()
server.start ()
return server
def stop_HTTP_Server (self):
conn = http.client.HTTPConnection (self.domain.strip ('/'))
conn.request ("QUIT", "/")
self.fileSys = HTTPServer.ret_fileSys ()
#self.fileSys = HTTPServer.ret_fileSys ()
conn.getresponse ()
#server.stop ()