Add support for HTTPS Servers

testenv/ChangeLog

@@ -1,3 +1,25 @@
+2014-01-02  Darshit Shah  <darnir@gmail.com>
+	* Makefile.am: Add new Test--https.py to list of tests and EXTRA_DIST.
+	Also replace all tabs with spaces in file for conformity.
+	* Test--https.py: New test to check if Wget works correctly with HTTPS
+	servers
+	* HTTPServer.py: Import new modules for use in HTTPS Servers
+	(HTTPSServer): New class that generates a SSL-wrapped socket for use in a
+	HTTPS Server.
+	(HTTPSd): HTTPS daemon class. Analogous to the HTTPd class
+	* WgetTest.py: Define global variables HTTP and HTTPS to reflect Server
+	types
+	(CommonMethods.exec_wget): Add the protocol information to the URL before
+	passing it to wget
+	(HTTPTest.__init__): Edit syntax. The servers variable now accepts a list of
+	servers defined by their type. E.g. HTTP, HTTPS.
+	(HTTPTest.Server_setup): Reflect change in type of variable servers.
+	However, we maintin the value of self.servers to allow most of the code to
+	remain unchanged.
+	(HTTPTest.init_HTTPS_Server): Initialize a HTTPS Server
+	* Test-Parallel-Proto.py: Edit to reflect slight change in Test Fiel Syntax.
+	* Test-Proto.py: Same
+
 2014-01-02  Darshit Shah  <darnir@gmail.com>
 
 	* WgetTest.py (CommonMentods.exec_wget): Wait for n seconds before calling

testenv/HTTPServer.py

@@ -1,10 +1,14 @@
 from http.server import HTTPServer, BaseHTTPRequestHandler
+from socketserver import BaseServer
 from posixpath import basename, splitext
 from base64 import b64encode
 from random import random
 from hashlib import md5
 import threading
+import socket
 import re
+import ssl
+import os
 
 
 class InvalidRangeHeader (Exception):
@@ -38,6 +42,23 @@ class StoppableHTTPServer (HTTPServer):
     def get_req_headers (self):
         return self.request_headers
 
+class HTTPSServer (StoppableHTTPServer):
+
+   def __init__ (self, address, handler):
+         BaseServer.__init__ (self, address, handler)
+         print (os.getcwd())
+         CERTFILE = os.path.abspath (os.path.join ('..', 'certs', 'wget-cert.pem'))
+         print (CERTFILE)
+         fop = open (CERTFILE)
+         print (fop.readline())
+         self.socket = ssl.wrap_socket (
+               sock = socket.socket (self.address_family, self.socket_type),
+               ssl_version = ssl.PROTOCOL_TLSv1,
+               certfile = CERTFILE,
+               server_side = True
+               )
+         self.server_bind ()
+         self.server_activate ()
 
 class WgetHTTPRequestHandler (BaseHTTPRequestHandler):
 
@@ -439,4 +460,8 @@ class HTTPd (threading.Thread):
     def server_sett (self, settings):
          self.server_inst.server_sett (settings)
 
+class HTTPSd (HTTPd):
+
+   server_class = HTTPSServer
+
 # vim: set ts=4 sts=4 sw=4 tw=80 et :

testenv/Makefile.am

@@ -51,6 +51,7 @@ TESTS = Test-auth-basic-fail.py             \
     Test-cookie.py                          \
     $(RACE_TESTS)                           \
     Test-Head.py                            \
+    Test--https.py							\
     Test-O.py                               \
     Test-Post.py                            \
     Test--spider-r.py
@@ -60,29 +61,30 @@ XFAIL_TESTS = Test-auth-both.py             \
 
 LOG_COMPILER = python3
 
-EXTRA_DIST = ColourTerm.py			\
-	FTPServer.py				\
-	HTTPServer.py				\
-	README					\
-	Test--spider-r.py			\
-	Test-Content-disposition-2.py		\
-	Test-Content-disposition.py		\
-	Test-Head.py				\
-	Test-O.py				\
-	Test-Parallel-Proto.py			\
-	Test-Post.py				\
-	Test-Proto.py				\
-	Test-auth-basic-fail.py			\
-	Test-auth-basic.py			\
-	Test-auth-both.py			\
-	Test-auth-digest.py			\
-	Test-auth-no-challenge-url.py		\
-	Test-auth-no-challenge.py		\
-	Test-auth-retcode.py			\
-	Test-auth-with-content-disposition.py	\
-	Test-c-full.py				\
-	Test-cookie-401.py			\
-	Test-cookie-domain-mismatch.py		\
-	Test-cookie-expires.py			\
-	Test-cookie.py				\
-	WgetTest.py
+EXTRA_DIST = ColourTerm.py          \
+    FTPServer.py                \
+    HTTPServer.py               \
+    README                  \
+    Test--spider-r.py           \
+    Test--https.py				\
+    Test-Content-disposition-2.py       \
+    Test-Content-disposition.py     \
+    Test-Head.py                \
+    Test-O.py               \
+    Test-Parallel-Proto.py          \
+    Test-Post.py                \
+    Test-Proto.py               \
+    Test-auth-basic-fail.py         \
+    Test-auth-basic.py          \
+    Test-auth-both.py           \
+    Test-auth-digest.py         \
+    Test-auth-no-challenge-url.py       \
+    Test-auth-no-challenge.py       \
+    Test-auth-retcode.py            \
+    Test-auth-with-content-disposition.py   \
+    Test-c-full.py              \
+    Test-cookie-401.py          \
+    Test-cookie-domain-mismatch.py      \
+    Test-cookie-expires.py          \
+    Test-cookie.py              \
+    WgetTest.py

testenv/Test--https.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+from sys import exit
+from WgetTest import HTTPTest, WgetFile, HTTPS, HTTP
+
+"""
+    This test ensures that Wget can download files from HTTPS Servers
+"""
+TEST_NAME = "HTTPS Downloads"
+############# File Definitions ###############################################
+File1 = "Would you like some Tea?"
+File2 = "With lemon or cream?"
+File3 = "Sure you're joking Mr. Feynman"
+
+A_File = WgetFile ("File1", File1)
+B_File = WgetFile ("File2", File2)
+C_File = WgetFile ("File3", File3)
+
+WGET_OPTIONS = "-d --no-check-certificate"
+WGET_URLS = [["File1", "File2"]]
+
+Files = [[A_File, B_File]]
+Existing_Files = [C_File]
+
+Servers = [HTTPS]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [A_File, B_File, C_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+                servers=Servers
+).begin ()
+
+exit (err)

testenv/Test-Parallel-Proto.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 from sys import exit
-from WgetTest import HTTPTest, WgetFile
+from WgetTest import HTTPTest, WgetFile, HTTP, HTTPS
 
 """
     This is a Prototype Test File for multiple servers.
@@ -22,7 +22,7 @@ WGET_URLS = [["File1"], ["File2"]]
 Files = [[A_File], [B_File]]
 Existing_Files = [C_File]
 
-no_of_servers = 2
+Servers = [HTTP, HTTP]
 
 ExpectedReturnCode = 0
 ExpectedDownloadedFiles = [A_File, B_File, C_File]
@@ -46,7 +46,7 @@ err = HTTPTest (
                 pre_hook=pre_test,
                 test_params=test_options,
                 post_hook=post_test,
-                servers=no_of_servers
+                servers=Servers
 ).begin ()
 
 exit (err)

testenv/Test-Proto.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 from sys import exit
-from WgetTest import HTTPTest, WgetFile
+from WgetTest import HTTPTest, WgetFile, HTTP, HTTPS
 
 """
     This is a Prototype Test File.
@@ -40,6 +40,8 @@ C_File = WgetFile ("File3", File3)
 WGET_OPTIONS = "-d --content-disposition --user=Sauron --password=TheEye"
 WGET_URLS = [["File1", "File2"]]
 
+Servers = [HTTP]
+
 Files = [[A_File, B_File]]
 Existing_Files = [C_File]
 
@@ -64,7 +66,8 @@ err = HTTPTest (
                 name=TEST_NAME,
                 pre_hook=pre_test,
                 test_params=test_options,
-                post_hook=post_test
+                post_hook=post_test,
+                server=Servers
 ).begin ()
 
 exit (err)

testenv/WgetTest.py

@@ -10,6 +10,9 @@ from subprocess import call
 from ColourTerm import printer
 from difflib import unified_diff
 
+HTTP = "HTTP"
+HTTPS = "HTTPS"
+
 """ A Custom Exception raised by the Test Environment. """
 
 class TestFailed (Exception):
@@ -57,7 +60,8 @@ class CommonMethods:
         cmd_line = WGET_PATH + " " + options + " "
         for i in range (0, self.servers):
             for url in urls[i]:
-                cmd_line += domain_list[i] + url + " "
+                protocol = "http://" if self.server_types[i] is "HTTP" else "https://"
+                cmd_line += protocol + domain_list[i] + url + " "
 #        for url in urls:
 #            cmd_line += domain_list[0] + url + " "
         print (cmd_line)
@@ -227,7 +231,7 @@ class HTTPTest (CommonMethods):
         pre_hook=dict(),
         test_params=dict(),
         post_hook=dict(),
-        servers=1
+        servers=[HTTP]
     ):
         try:
             self.Server_setup (name, pre_hook, test_params, post_hook, servers)
@@ -246,13 +250,14 @@ class HTTPTest (CommonMethods):
 
     def Server_setup (self, name, pre_hook, test_params, post_hook, servers):
         self.name = name
-        self.servers = servers
+        self.server_types = servers
+        self.servers = len (servers)
         printer ("BLUE", "Running Test " + self.name)
         self.init_test_env (name)
         self.server_list = list()
         self.domain_list = list()
-        for server_number in range (0, servers):
-            server_inst = self.init_HTTP_Server ()
+        for server_type in servers:
+            server_inst = getattr (self, "init_" + server_type + "_Server") ()
             self.server_list.append (server_inst)
             domain = self.get_domain_addr (server_inst.server_address)
             self.domain_list.append (domain)
@@ -301,6 +306,11 @@ class HTTPTest (CommonMethods):
         server.start ()
         return server
 
+    def init_HTTPS_Server (self):
+        server = HTTPServer.HTTPSd ()
+        server.start ()
+        return server
+
     def stop_HTTP_Server (self):
         self.Request_remaining = list ()
         for server in self.server_list:

testenv/certs/wget-cert.pem

@@ -0,0 +1,30 @@
+-----BEGIN PRIVATE KEY-----
+MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBAMV8qEpuSVUdWaAY
+F2N1ljGEJ/907Og5B0aZLeDskmLAOohKMWTiiSx+lseXVD/Zf/LaFfy/+q0Rk5+o
+pFEPEEjadvdxogb9HPwjfj48ng74yV1c5ZGRx/aIeIJN9cacfs4J5NlT3ZPiV8/2
+mpBurBYvta5tneUl+lx4NHTEBmjTAgMBAAECgYBHlFlDMRovWYYEuvavPA2GQQpm
+UzETMqhqdFbmsZiVZmtQvuOMV3e0wuVPzo/g3Kq9kUJq7AKl/DrvoaZ9IuKZgkDD
+0QEBYo/lcxEA9qcfgVs5XLp9ED1mXzJSZ3bmpCDqa2NjG7yFdWzPxc1DXmT05MrF
+bZbb0Wao0tvMwoeJYQJBAOql5uOyjDHvLLuS0IFKbYz4LQwAp7Gjs0ZS9qLNhQQn
+m5Vr8xS9QwFID693K6aDl3tqSCIwSnyInacj8M8v18sCQQDXdReE2i4LKOVLcQsP
+XabN96fFLlnoIh9MqFza4skjhXJWqjBLgJuFqyT5CTbU9TmaoIPXdo4454P1CCgR
+KEIZAkAZE7nlQ8Ov4nvJYBtgde/XTP6jdb52QaR7M4qgQ46frwv1oB/Oa5upm2Xx
+vq6vkQiza9xhqv+K557RqgmmWtqZAkASoXJmL4OZvXCOZHkDXCLHXqnoOAjYNNMm
+Csz0tHWWF7z6V38TmExac6Ef07clFQtlHoooAH1t2D8l2g205hlJAkBfeghbZDdY
+16NtVnvtzjjhKqZFqwTSANFV8NSzgb/QiNnX0hsMPt9bbc5VCo77Ly2oP5SvixfZ
+kjrIQqDV8MLu
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICODCCAaGgAwIBAgIJAOiSkPuPcAwqMA0GCSqGSIb3DQEBBQUAMDUxCzAJBgNV
+BAYTAklOMRMwEQYDVQQIDApTb21lLVN0YXRlMREwDwYDVQQKDAhHTlUgV2dldDAe
+Fw0xMzEyMDcwNTA3NTRaFw0xNDEyMDcwNTA3NTRaMDUxCzAJBgNVBAYTAklOMRMw
+EQYDVQQIDApTb21lLVN0YXRlMREwDwYDVQQKDAhHTlUgV2dldDCBnzANBgkqhkiG
+9w0BAQEFAAOBjQAwgYkCgYEAxXyoSm5JVR1ZoBgXY3WWMYQn/3Ts6DkHRpkt4OyS
+YsA6iEoxZOKJLH6Wx5dUP9l/8toV/L/6rRGTn6ikUQ8QSNp293GiBv0c/CN+Pjye
+DvjJXVzlkZHH9oh4gk31xpx+zgnk2VPdk+JXz/aakG6sFi+1rm2d5SX6XHg0dMQG
+aNMCAwEAAaNQME4wHQYDVR0OBBYEFLhtTG9a6v3ihL5DeWKfq6doYI42MB8GA1Ud
+IwQYMBaAFLhtTG9a6v3ihL5DeWKfq6doYI42MAwGA1UdEwQFMAMBAf8wDQYJKoZI
+hvcNAQEFBQADgYEApTEZX3cgmgdXDJsu7wtkejtq3vuyi6NXBUlHzoYzWaS5wn8P
+uDG4G9zd1cwmwrbYA8lS+ANWvkcqjM68gMs1ARMZRS0IrYMCN8bokQw+16sqImZO
+THX50Sb5U+9e1IotDWyRBNO10znsoh569BxhJ5WZdIaoKHOJdXEYV+3Y/hg=
+-----END CERTIFICATE-----