github-repos/wget
1
0
Fork 0
mirror of https://github.com/mirror/wget.git synced 2025-03-26 11:42:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into parallel-wget

Browse Source 
Conflicts:
	src/http.c
	testenv/Makefile.am
	testenv/Test--https.py
	testenv/Test--spider-r.py
	testenv/Test-Content-disposition-2.py
	testenv/Test-Content-disposition.py
	testenv/Test-Head.py
	testenv/Test-O.py
	testenv/Test-Parallel-Proto.py
	testenv/Test-Post.py
	testenv/Test-Proto.py
	testenv/Test-auth-basic-fail.py
	testenv/Test-auth-basic.py
	testenv/Test-auth-both.py
	testenv/Test-auth-digest.py
	testenv/Test-auth-no-challenge-url.py
	testenv/Test-auth-no-challenge.py
	testenv/Test-auth-retcode.py
	testenv/Test-auth-with-content-disposition.py
	testenv/Test-c-full.py
	testenv/Test-cookie-401.py
	testenv/Test-cookie-domain-mismatch.py
	testenv/Test-cookie-expires.py
	testenv/Test-cookie.py
	testenv/conf/__init__.py
	testenv/conf/authentication.py
	testenv/conf/expect_header.py
	testenv/conf/expected_files.py
	testenv/conf/expected_ret_code.py
	testenv/conf/files_crawled.py
	testenv/conf/hook_sample.py
	testenv/conf/local_files.py
	testenv/conf/reject_header.py
	testenv/conf/response.py
	testenv/conf/send_header.py
	testenv/conf/server_files.py
	testenv/conf/urls.py
	testenv/conf/wget_commands.py
	testenv/misc/colour_terminal.py
	testenv/server/http/http_server.py
	testenv/test/base_test.py
	testenv/test/http_test.py
This commit is contained in:
Darshit Shah 2014-09-14 16:57:15 +05:30
commit af702340a1
51 changed files with 919 additions and 154 deletions

91
.gitignore vendored Normal file
View File

@ -0,0 +1,91 @@
# Project Root
/.sc-start-sc_bindtextdomain
/.sc-start-sc_prohibit_HAVE_MBRTOWC
/.version
/GNUmakefile
/INSTALL
/Makefile
/Makefile.in
/aclocal.m4
/autom4te.cache/
/config.cache
/config.log
/config.status
/configure
/lib/
/maint.mk
# Wildcard Ignores
*~
po/*.gmo*
po/*.po*
src/*.o
testenv/*.log
testenv/*.trs
# build-aux/
build-aux/.gitignore
build-aux/compile
build-aux/config.guess
build-aux/config.sub
build-aux/depcomp
build-aux/install-sh
build-aux/mdate-sh
build-aux/missing
build-aux/snippet/
build-aux/test-driver
build-aux/texinfo.tex
# doc/
doc/Makefile
doc/Makefile.in
doc/sample.wgetrc.munged_for_texi_inclusion
doc/stamp-vti
doc/version.texi
doc/wget.1
doc/wget.info
doc/wget.pod
# m4/
m4/.gitignore
m4/gnulib-cache.m4
# po/
po/.gitignore
po/.reference/
po/LINGUAS
po/Makefile
po/Makefile.in
po/Makevars
po/POTFILES
po/wget.pot
# src/
src/.deps/
src/Makefile
src/Makefile.in
src/build_info.c
src/config.h
src/config.h.in
src/cscope.out
src/css.c
src/css_.c
src/libunittest.a
src/stamp-h1
src/tags
src/version.c
src/wget
# testenv/
testenv/conf/__pycache__/
testenv/exc/__pycache__/
testenv/misc/__pycache__/
testenv/server/__pycache__/
testenv/server/http/__pycache__/
testenv/test/__pycache__/
testenv/Makefile
testenv/Makefile.in
# tests/
tests/Makefile
tests/Makefile.in
tests/WgetTest.pm
tests/unit-tests
# util/
util/Makefile
util/Makefile.in
# others
/tags
/cscope.out

17
ChangeLog
View File

@ -1,3 +1,20 @@
2014-07-25 Darshit Shah <darnir@gmail.com>
* .gitignore: Add a gitignore file for the project.
2013-08-31 Darshit Shah <darnir@gmail.com>
* configure.ac: Add testenv/Makefile to AC_CONFIG_FILES.
* Makefile.am: Add testenv to SUBDIRS
2014-07-22 Darshit Shah <darnir@gmail.com>
* configure.ac: Fix broken code for detecting libpsl
2014-07-21 Darshit Shah <darnir@gmail.com>
* configure.ac: Fix check for Libpsl
2014-06-28 Giuseppe Scrivano <gscrivan@redhat.com>
* cfg.mk (local-checks-to-skip): Remove some checks.

15
configure.ac
View File

@ -61,9 +61,15 @@ dnl
dnl Process features.
dnl
ENABLE_PSL=no
AC_ARG_WITH(libpsl,
AS_HELP_STRING([--without-libpsl],
[disable support for libpsl cookie checking.]))
[disable support for libpsl cookie checking.]),
[],
[AC_SEARCH_LIBS(psl_builtin, psl,
[ENABLE_PSL=yes; AC_DEFINE([HAVE_LIBPSL], [1], [PSL Support Enabled])],
[AC_MSG_WARN(*** libpsl not found. Falling back to Wget builtin cookie checking.)])
])
AC_ARG_WITH(ssl,
[[ --without-ssl disable SSL autodetection
@ -250,11 +256,6 @@ dnl
dnl Checks for libraries.
dnl
AS_IF([test x"$with_libpsl" != xno], [
with_libpsl=yes
AC_CHECK_LIB([psl], [psl_builtin])
])
AS_IF([test x"$with_zlib" != xno], [
with_zlib=yes
AC_CHECK_LIB(z, compress)
@ -675,7 +676,7 @@ AC_MSG_NOTICE([Summary of build options:
Libs: $LIBS
SSL: $with_ssl
Zlib: $with_zlib
PSL: $with_libpsl
PSL: $ENABLE_PSL
Digest: $ENABLE_DIGEST
NTLM: $ENABLE_NTLM
OPIE: $ENABLE_OPIE

4
doc/ChangeLog
View File

@ -1,3 +1,7 @@
2014-08-03 Giuseppe Scrivano <gscrivano@gnu.org>
* wget.texi (Download Options): Fix texinfo warning.
2014-06-11 Giuseppe Scrivano <gscrivan@redhat.com>
* Makefile.am: Remove @VAR@ with $FOO.

11
doc/wget.texi
View File

@ -956,7 +956,8 @@ Don't use proxies, even if the appropriate @code{*_proxy} environment
variable is defined.
@c man end
@xref{Proxies} for more information about the use of proxies with Wget.
@xref{Proxies}, for more information about the use of proxies with
Wget.
@c man begin OPTIONS
@cindex quota
@ -1306,8 +1307,8 @@ really important, do not leave them lying in those files either---edit
the files and delete them after Wget has started the download.
@iftex
@xref{Security Considerations} for more information about security issues
with Wget.
@xref{Security Considerations}, for more information about security
issues with Wget.
@end iftex
@cindex Keep-Alive, turning off
@ -1843,8 +1844,8 @@ really important, do not leave them lying in those files either---edit
the files and delete them after Wget has started the download.
@iftex
@xref{Security Considerations} for more information about security issues
with Wget.
@xref{Security Considerations}, for more information about security
issues with Wget.
@end iftex
@cindex .listing files, removing

17
src/ChangeLog
View File

@ -2,6 +2,23 @@
* Makefile.am (wget_SOURCES): Remove space-tab indentation.
2014-07-23 Darshit Shah <darnir@gmail.com>
* http.c (gethttp): Fix a memory leak when retrying authorization
(gethttp): Fix memory leak when trying to parse content disposition headers
(http_loop): Assigning a new value to *local)file without freeing the old
one causes a memory leak
(http_loop): Free the HTTP message and error strings before continuing loop
2014-07-21 Daniel Stenberg <daniel@haxx.se>
* main.c (print_help): HTTP Method is a part of the Request not Header
2014-07-21 Darshit Shah <darnir@gmail.com>
* cookies.c (check_domain_match): Fix a potential memory leak when checking
cookie domain names
2014-07-07 Tomas Hozza <thozza@redhat.com>
* iri.c (locale_to_utf8): Fix checking of iconv_open return code.

5
src/cookies.c
View File

@ -546,9 +546,12 @@ check_domain_match (const char *cookie_domain, const char *host)
xfree (cookie_domain_lower);
xfree (host_lower);
return true ? (is_acceptable == 1) : false;
return is_acceptable == 1;
no_psl:
/* Cleanup the PSL pointers first */
xfree (cookie_domain_lower);
xfree (host_lower);
#endif
/* For efficiency make some elementary checks first */

10
src/http.c
View File

@ -2702,6 +2702,7 @@ read_header:
resp_free (resp);
xfree (head);
xfree (auth_stat);
xfree (hs->message);
goto retry_with_auth;
}
else
@ -2762,6 +2763,8 @@ read_header:
local_file));
hs->local_file = url_file_name (u, local_file);
}
xfree_null (local_file);
}
/* TODO: perform this check only once. */
@ -3726,6 +3729,8 @@ Remote file exists.\n\n"));
got_name = true;
*dt &= ~HEAD_ONLY;
count = 0; /* the retrieve count for HEAD is reset */
xfree_null (hstat.message);
xfree_null (hstat.error);
continue;
} /* send_head_first */
} /* !got_head */
@ -3873,7 +3878,10 @@ Remote file exists.\n\n"));
exit:
if (ret == RETROK && local_file && !(*local_file))
*local_file = xstrdup (hstat.local_file);
{
xfree_null (*local_file);
*local_file = xstrdup (hstat.local_file);
}
free_hstat (&hstat);
return ret;

2
src/main.c
View File

@ -636,7 +636,7 @@ HTTP options:\n"),
N_("\
--post-file=FILE use the POST method; send contents of FILE.\n"),
N_("\
--method=HTTPMethod use method \"HTTPMethod\" in the header.\n"),
--method=HTTPMethod use method \"HTTPMethod\" in the request.\n"),
N_("\
--body-data=STRING Send STRING as data. --method MUST be set.\n"),
N_("\

502
testenv/ChangeLog
View File

@ -454,3 +454,505 @@
* Test-auth-no-challenge.py: Ensure --auth-no-challenge is honoured
* Test-auth-retcode.py: Ensure correct return code after 403 Forbidden
response.
2014-08-08 Darshit Shah <darnir@gmail.com>
* conf/__init__.py: Add extra newline according to PEP8
* conf/{authentication,expect_header,expected_files,expected_ret_code,
files_crawled,hook_sample,local_files,reject_header,response,send_header,
server_files,urls,wget_commands}.py: Add docstrings explaining the conf file
and how it should be used
* server/http/http_server (InvalidRangeHeader): Clear TODO and eliminate
this exception. Use ServerError for all such purposes.
(_Handler): Remove reference to InvalidRangeHeader
(_handler.parse_range_header): User ServerError instead of InvalidRangeHeader
(_Handler.do_GET): Add docstring
(_Handler.do_POST): Add docstring. Also create an empty dict for rules if
no rules are supplied. Send the Location header as suggested in RFC 7231
(_Handler.do_PUT): Don't pop the server file already. Push it to later in ..
(_Handler.send_put): .. Here. If the file exists respond with a 204 No
Content message and pop the file for replacement. Do not send the
Content-Length, Content-Type headers since PUT requests should not respond
with data.
(_Handler.parse_auth_header): Fit line within 80 chars
(_Handler.check_response): Better visual indent
(_Handler.authorize_digest): Better visual indent.
(_Handler.expect_headers): Remove unused function
(_Handler.guess_type): Fix indentation
(HTTPd): Add newline according to PEP8 guidelines
(HTTPSd): Fix indentation
(StoppableHTTPServer): Add docstring
(HTTPSServer): Fix indentation
(WgetHTTPRequestHandler): Merge class into _handler.
(_Handler): Add docstring
(_Handler.parse_range_header): Fix indentation
(ServerError): Split exception into separate file ...
* exc/server_error.py: ... Here
* misc/colour_terminal.py: Add docstring, fix indentation
* test/base_test.py: Fix visual indent
* test/http_test.py: Fit within 80 char lines
2014-08-04 Darshit Shah <darnir@gmail.com>
* conf/server_conf.py: Delete file. Server configuration is now done via the
server_conf() method.
* server/http/http_server.py (StppableHTTPServer.server_sett): Delete
method required by the above hook
(HTTPd.server_sett): Same
2014-07-26 Darshit Shah <darnir@gmail.com>
* Test-*.py: Remove the '-d' switch from WGET_OPTIONS.
* test/base_test (BaseTest.gen_cmd_line): Add --debug and --no-config to the
list of switches passed to wget unconditionally.
2014-07-23 Darshit Shah <darnir@gmail.com>
* test/base_test.py (BaseTest.gen_cmd_line): Add support for running all
tests through valgrind if the relevant environment variable is set
* conf/expected_ret_code (ExpectedRetCode.__call__): Valgrind returns error
code 45 when it detects a memory leak.
* Readme: Update with details about valgrind tests
2014-07-22 Darshit Shah <darnir@gmail.com>
* (README): Remove old TODO and document SERVER_WAIT variable
2014-06-22 Darshit Shah <darnir@gmail.com>
* (conf.files_crawled): diff is a set object and needs explicit str
conversion.
2014-03-13 Zihang Chen <chsc4698@gmail.com>
* base_test.py:
(CommonMethods): Rename to BaseTest.
(BaseTest): Implement __init__ method where the class-wide variables are
initialized. Also variable names like `xxx_list` is renamed to its plural
form, e.g. `server_list` => `servers`.
(BaseTest.init_test_env): Remove name argument due to its unnecessarity.
(BaseTest.get_test_dir): Because the path of the test directory is needed
in multiple methods, this method is implemented.
(BaseTest.get_domain_addr): Rewrite the return statement utilizing str
formatting (which is more Pythonic).
(BaseTest.get_cmd_line): Rename to gen_cmd_line. Change the variables with
capitcal characters to lower ones. Also, the nested for loop is rewritten
to a plain loop using the zip function.
(BaseTest.__gen_local_filesys): Rename to gen_local_fs_snapshot. Move to
ExpectedFiles in conf/expected_files.py and is marked as a static
method. Refactor to a less verbose implementation.
(BaseTest._check_downloaded_files): Rename to __call__ to agree with the
invocation in test case classes. Move to ExpectedFiles in
conf/expected_files.py.
(BaseTest.get_server_rules): Refactor to a more Pythonic form utilizing
dict.items() and is marked static.
(BaseTest.stop_server): (new method) an abstract method which should stop
the currently using servers.
(BaseTest.instantiate_server_by): (new method) an abstract method which
should instantiate a server instance according to the given argument.
(BaseTest.__enter__): (new method) method which initialize the context
manager
(BaseTest.__exit__): (new method) method that finilize the context manager
and deal with the exceptions during the execution of the with statement,
subclasses can override this method for extensibility
* http_test.py:
(HTTPTest.__init__): Add call to super.__init__. Default values of
pre_hook, test_params, post_hook are set to None to avoid a subtle bug of
Python. Argument servers is renamed to protocols.
(HTTPTest.Server_setup): Move to BaseTest and rename to server_setup.
Calls to pre_hook_call, call_test, post_hook_call are removed.
(HTTPTest.hook_call, pre_hook_call, call_test, post_hook_call): Move to
BaseTest for that both HTTP test cases and FTP test cases may use these
methods.
(HTTPTest.init_HTTP_Server, init_HTTPS_Server): Merge and rename to
instantiate_server_by to implement the abstract method in BaseTest.
(HTTPTest.stop_HTTP_Server): Rename to stop_server to implement the
abstract method in BaseTest. Also, pull out the part where remaining
requests are gathered into a new method request_remaining.
(BaseTest.act_retcode): Rename to ret_code because ExpectedRetCode is
moved out from BaseTest, so the name act_retcode is actually a bit
verbose.
* conf/expected_ret_code.py:
(ExpectedRetCode.__call__): Rewrite the str into a more readable form.
* conf/files_crawled.py:
(FilesCrawled.__call__): Refactor this method into a more Pythonic form
utilizing the zip function.
* conf/local_files.py:
(LocalFiles__call__): Rewrite this method with the recommended with
statement.
* conf/server_conf.py:
(ServerConf.__call__): Rewrite this method due to BaseTest.server_list is
renamed to BaseTest.servers.
* conf/server_files.py:
(ServerFiles.__call__): Refactor the nested for loop into a plain one
utilizing the zip function.
* conf/urls.py:
(URLs): Rename url_list to urls.
* conf/wget_commands.py:
(WgetCommands): Rename command_list to commands, rename test_obj.options
to test_obj.wget_options.
* Test--https.py, Test-Proto.py, Test-Parallel-Proto.py: Argument servers
is changed to protocols due to change in the signature of
HTTPTest.__init__.
2014-03-13 Zihang Chen <chsc4698@gmail.com>
* test: (new package) package for test case classes
* WgetTest.py: Split into test/base_test.py and test/http_test.py.
* Test-*.py: Optimize the imports according to changes of WgetTest.py
2014-03-13 Zihang Chen <chsc4698@gmail.com>
* server: (new package) package for the server classes
* server.http: (new package) package for HTTP server
* server.ftp: (new package) package for FTP server
* HTTPServer.py: Move to server/http/http_server.py. Also change the
CERTFILE to '../certs/wget-cert.pem'.
* FTPServer.py: Move to server/ftp/ftp_server.py.
* WgetTest.py: Optimize import respect to the server classes.
2014-03-13 Zihang Chen <chsc4698@gmail.com>
* conf: (new package) package for rule classes and hook methods
* WgetTest.py:
(CommonMethods.Authentication): Move to conf/authentication.py.
(CommonMethods.ExpectHeader): Move to conf/expect_header.py.
(CommonMethods.RejectHeader): Move to conf/reject_header.py.
(CommonMethods.Response): Move to conf/response.py.
(CommonMethods.SendHeader): Move to conf/send_header.py.
(CommonMethods.ServerFiles): Move to conf/server_files.py.
(CommonMethods.LocalFiles): Move to conf/local_files.py.
(CommonMethods.ServerConf): Move to conf/server_conf.py.
(CommonMethods.WgetCommands): Move to conf/wget_commands.py.
(CommonMethods.Urls): Move to conf/urls.py.
(CommonMethods.ExpectedRetcode): Move to conf/expected_retcode.py.
(CommonMethods.ExpectedFiles): Move to conf/expected_files.py.
(CommonMethods.FilesCrawled): Move to conf/files_crawled.py.
(CommonMethods.__check_downloaded_files): Rename to
_check_downloaded_files, so that the method is callable from outside the
class.
(CommomMethods.get_server_rules): Modify so that it utilizes the conf
package.
(HTTPTest): Add a method hook_call(configs, name) to reduce duplications
in pre_hook_call, call_test and post_hook_call utilizing the conf package.
* conf/hook_sample.py: (new file) sample for hooks
* conf/rule_sample.py: (new file) sample for rules
* REAMDE: Update sections about customizing rules and hooks.
2014-03-13 Zihang Chen <chsc4698@gmail.com>
* exc: (new package) package for miscellaneous exceptions
* WgetTest.py: Move TestFailed to exc/test_failed.py.
2014-03-13 Zihang Chen <chsc4698@gmail.com>
* Test-Proto.py: Fix a typo (line 71: server to servers).
2014-03-13 Zihang Chen <chsc4698@gmail.com>
* WgetTest.py: Move WgetFile to package misc.
* README: Modify documentation respect to WgetFile.
* Test-*.py: Optimize imports about WgetFile.
2014-03-13 Zihang Chen <chsc4698@gmail.com>
* misc: (new package) package for miscellaneous modules
* ColourTerm.py: Move to package misc and rename to colour_terminal.py,
add print_color functions to reduce the use of string literals like
"BLUE", "RED" etc.
* WgetTest.py:
(CommonMethods.Server_setup): Change invocation to printer to print_blue.
(CommonMethods.FilesCrawled): Change invocation to printer to print_red.
(HTTPTest.__init__): Change invocations to printer to print_red and
print_green respectively.
2014-01-02 Darshit Shah <darnir@gmail.com>
* Makefile.am: Add new Test--https.py to list of tests and EXTRA_DIST.
Also replace all tabs with spaces in file for conformity.
* Test--https.py: New test to check if Wget works correctly with HTTPS
servers
* HTTPServer.py: Import new modules for use in HTTPS Servers
(HTTPSServer): New class that generates a SSL-wrapped socket for use in a
HTTPS Server.
(HTTPSd): HTTPS daemon class. Analogous to the HTTPd class
* WgetTest.py: Define global variables HTTP and HTTPS to reflect Server
types
(CommonMethods.exec_wget): Add the protocol information to the URL before
passing it to wget
(HTTPTest.__init__): Edit syntax. The servers variable now accepts a list of
servers defined by their type. E.g. HTTP, HTTPS.
(HTTPTest.Server_setup): Reflect change in type of variable servers.
However, we maintin the value of self.servers to allow most of the code to
remain unchanged.
(HTTPTest.init_HTTPS_Server): Initialize a HTTPS Server
* Test-Parallel-Proto.py: Edit to reflect slight change in Test Fiel Syntax.
* Test-Proto.py: Same
2014-01-02 Darshit Shah <darnir@gmail.com>
* WgetTest.py (CommonMentods.exec_wget): Wait for n seconds before calling
the Wget executable.
2013-12-27 Darshit Shah <darnir@gmail.com>
* WgetTest.py: Add modeline
(CommonMethods.ServerConf): New pre-test hook that sets
BaseHTTPRequestHandler class variables in all available servers
* HTTPServer.py (HTTPd.ServerConf): Call the respective method in the Server
to set the class variables
(StoppableHTTPServer.server_sett): Set the handler class variables
2013-12-26 Darshit Shah <darnir@gmail.com>
* WgetTest.py (HTTPTest.call_test): Correct the call to stop_HTTP_Server.
2013-12-25 Darshit Shah <darnir@gmail.com>
* WgetTest.py (CommonMehtods.exec_wget): Catch and handle exception if the
Wget executable is not found at src/wget
(HTTPTest.call_test): In case of error during execution, remove all existing
servers before quitting
2013-12-15 Darshit Shah <darnir@gmail.com>
* WgetTest.py (HTTPTest.HTTP_setup): Rename to Server_setup so it can be
easily reused for other non-HTTP servers.
(HTTPTest.__init__): Call Server_setup instead of HTTP_setup
(HTTPTest.Server_setup): Split into three more functions, that handle
pre-hooks, test execution and post-hooks respectively.
(HTTPTest.pre_hook_call): Set up and execute the pre-test hooks. Code split
from HTTPTest.Server_setup
(HTTPTest.call_test): Execute wget and log exit code. Code split from
HTTPTest.Server_setup
(HTTPTest.post_hook_call): Set up and execute post-test hooks. Code split
from HTTPTest.Server_setup
2013-10-14 Giuseppe Scrivano <gscrivan@redhat.com>
* Makefile.am (XFAIL_TESTS): Remove Test--spider-r.py.
2013-10-06 Giuseppe Scrivano <gscrivan@redhat.com>
* Makefile.am (EXTRA_DIST): Distribute test files.
2013-09-16 Darshit Shah <darnir@gmail.com>
* README: Update documentation
2013-09-14 Darshit Shah <darnir@gmail.com>
* HTTPServer.py (StoppableHTTPServer): Define object variable
request_headers which stores a list of requests received by the server
(StoppableHTTPServer.get_req_headers): Return the list of Request
Headers stored by the server
(_Handler.do_HEAD): Send the Request MEthod string for identification
(_Handler.do_GET): Same
(_Handler.__log_request): Log the request in Request_Headers list
(_Handler.send_head): Make a call to __log_request
* Test--spider-r.py: Add new list, Request_List, which contains all
the requests that Wget is expected to send. This will allow for
fine-grained tests on recursive downloading.
* WgetTest.py (CommonMethods.FilesCrawled): New Post-Test Hook, that
ensures that all the expected Files on the server were accessed as
expected.
(HTTPTest.stop_HTTP_server): On stopping server, asks it to respond
with list of all requests it received.
2013-09-13 Darshit Shah <darnir@gmail.com>
* Test--spider-r.py: Test retrieval in recursive spider mode.
* Makefile.am: add new file
2013-09-13 Darshit Shah <darnir@gmail.com>
* HTTPServer.py (_Handler.do_HEAD): If requested path is /, respond
with /index.html
(_Handler.do_HEAD): Smartly guess value of Content-Type Header from
file extension
(_Handler.guess_type): Use a preset list of extensions and
Content-Type strings. If the extension matches one in the list, use
that string, else default to "text/plain"
2013-09-13 Darshit Shah <darnir@gmail.com>
* WgetTest.py (CommonMethods._replace_substring): New method that will
replace a substring delimited by {{ }} characters by the value of
self.<substring> variable
(CommonMethods.WgetCommands): Use the _replace_substring () call to
replace the substrings in the the command line.
(CommonMethods.ServerFiles): Run the _replace_substring () method on
the File contents too.
2013-09-11 Darshit Shah <darnir@gmail.com>
* WgetTest.py (CommonMethods.exec_wget): Expect domain_list instead of
domain.
(CommonMethods.get_cmd_line): Same. Generate command line by
prepending to each file it's respective domain string
(CommonMethods.ServerFiles): Generate file_list and server_rules for
each Server and set the config details
(HTTPTest): New named parameter, servers which signifies number of
servers to spawn
(HTTPTest.HTTP_setup): This method now takes servers as a new
parameter. Instead of storing server and domain, we now store
server_list and domain_list. Each server must be initialized through a
loop.
(HTTPTest.stop_HTTP_server): Stop all servers in a loop.
* Test-Parallel-Proto.py: Prototype test file for multiple servers.
2013-09-10 Darshit Shah <darnir@gmail.com>
* WgetTest.py (HTTPTest.stop_HTTP_server): With the threaded servers,
we can simply use the socketserver.shutdown() method to close the
server instead of sending a QUIT command
* HTTPServer.py (StoppabelHTTPServer.serve_forever): Delete method. No
need to override this method anymore.
(WgetHTTPRequestHandler.do_QUIT): No longer required
(HTTPd): Rename self.server to self.server_inst to reduce ambiguity
when referenced from WgetTest
2013-09-08 Darshit Shah <darnir@gmail.com>
* README (File Structure): Add explanation about various variables
used consistently across all tests.
2013-09-07 Darshit Shah <darnir@gmail.com>
* HTTPServer.py: Remove bunch of old code artefacts
* WgetTest.py: Same
2013-09-07 Darshit Shah <darnir@gmail.com>
* HTTPServer.py (StoppableHTTPServer.server_conf): Change global
variable fileSys to an object variable. This is good programming
practice and required for parallel-wget support.
(StoppableHTTPServer.server_forever): Edit overridden method to remove
the global queue variable. No longer required under the new working
(WgetHTTPRequestHandler.do_QUIT): Don't push fileSys through the queue
(_Handler): Rename class __Handler to _Handler to match Python's
encapsulation rules
(_Handler.do_POST): fileSys is now an object variable of the server
(_Handler.do_PUT): Same
(_Handler.send_put): Same
(_Handler.send_head): Same
(HTTPd): New class that wraps around the server for Threading
(create_server): Make new object of HTTPd.
(spawn_server): Start the thread created through create_server
(ret_fileSys): Removed method. No longer required.
* WgetTest.py (HTTPTest.__init__): Don't explicitly set
self.act_retcode. Instead toggle tests_passed boolean to set the
correct return code.
(HTTPTest.HTTP_setup): We no longer call HTTPServer.spawn_server to
start a new instance of the server.
(HTTPTest.init_HTTP_server): We no longer call the old
create_server(), spawn_server() methods. Instead use the new HTTPd
class interface to create new instances of the server
(HTTPTest.stop_HTTP_server): Don't ask server to return fileSys.
2013-09-07 Darshit Shah <darnir@gmail.com>
* Test-Post.py: Test basic functionality for sending HTTP POST
requests using the --method command
* Makefile.am: Add new test
2013-09-06 Darshit Shah <darnir@gmail.com>
* WgetTest.py (CommonMethods.__check_downloaded_files): Print a
unified diff in case there is a mismatch in the file contents
2013-09-06 Darshit Shah <darnir@gmail.com>
* HTTPServer.py (WgetHTTPRequestHandler.test_cookies): Comment out the
old test_cookies code. This is no longer used and was causing problems
with expected cookies. The code will soon be removed anyways
* Test-cookie.py: Add new test for basic cookie functionality
* Test-cookie-401.py: Ensure cookies are saved during a 401 response
* Test-cookie-expires.py: Ensure that the Expires field is correctly
handled
* Test-cookies-domain-mismatch.py: Ensure that mismatched domains are
handled by Wget
* Makefile.am: Add the new tests
2013-09-06 Darshit Shah <darnir@gmail.com>
* README: New section on pending work. Will keep updating this to keep
track of work that remains to be done on this implementation
2013-09-05 Darshit Shah <darnir@gmail.com>
* Test-auth-with-content-disposition.py: Add test that ensures Content
Disposition works alongwith authentication
* Makefile.am: Add new test
2013-09-04 Darshit Shah <darnir@gmail.com>
* Test-c-full.py: Test Continue options
* Makefile.am: Add Test-c-full.py and Test-O
2013-09-02 Darshit Shah <darnir@gmail.com>
* Makefile.am: Add new Test
* Test-Head.py: New Test to ensure HEAD requests are handled correctly
2013-08-31 Darshit Shah <darnir@gmail.com>
* README: Explain that TEST_NAME needs to be unique
* Test-auth-no-challenge.py: Edit non-unique TEST_NAME
2013-08-31 Darshit Shah <darnir@gmail.com>
* HTTPTest.py (ServerError): Define new Exception for handling
internal control flow.
(StoppableHTTPServer.SendHeader): Simply pass. Do nothing. Adding
functionality here seems to crash for no apparent reason.
(stoppableHTTPServer.send_cust_headers): Minor optimization. No need
for extra variable.
(__Handler.Response): Handle explicit Response Code Rules
(__Handler.Authentication): Handle Authentication rules
(__Handler.handle_auth): Actual worker method for authentication
(__Handler.ExpectHeader): Ensure Expected Headers are received
(__Handler.RejectHeader): Ensure Blacklisted Headers are NOT received
(__Handler.send_HEAD): Dynamically call server rule functions based on
the self.rules list. This feature will later be added to POST/PUT, etc
2013-08-31 Darshit Shah <darnir@gmail.com>
* WgetTest.py: Remove import module defaultdict.
(CommonMethods.get_server_rules): server_rules should be a dict, not a
defaultdict (list).
* HTTPServer.py (WgetHTTPRequestHandler.get_rule_list): If rule does
not exist, return None. Not an emppty list.
(WgetHTTPRequestHandler.test_cookies): Rule variable is not a list
(__Handler.send_cust_headers): Same
(__Handler.custom_response): Same
(__Handler.is_authorized): Same
(__Handler.expect_headers): Same
(__Handler.reject_headers): Same
2013-08-31 Darshit Shah <darnir@gmail.com>
* README: (newfile) Simple help / instructions about using the Test
Environment.
* Makefile.am: (newfile) Makefile for the Test Environment. Uses the
Automake Parallel Test Harness
* WgetTest.py: (newfile) Base module that executes the Test.
* HTTPServer.py: (newfile) Contains the custom HTTP Server for the
Test Environment. Creates an instance of http.server in Python3.
* FTPServer.py: (newfile) Overrides methods from pyftpdlib for use in
the Test Environment. ** Work under progress **.
* ColourTerm.py: (newfile) A custom module to output coloured text to
the terminal. Known to work on POSIX shells.
* Test-Proto.py: (newfile) A prototype Test File. This should be
copied when writing a new Test Case.
* Test-Content-disposition-2.py: Test Content Disposition clobbering
* Test-Content-disposition.py: Test Content Disposition Headers
* Test-O.py: Test Output filename command
* Test-auth-basic-fail.py: Test returncode on auth failure
* Test-auth-basic.py: Test Basic Auth negotiation
* Test-auth-both.py: Test handling of Multiple auth providers. This
test currently fails.
* Test-auth-digest.py: Test Digest Auth Negotiation
* Test-auth-no-challenge-url.py: Ensure --auth-no-challenge is handled
when auth details are in-URL.
* Test-auth-no-challenge.py: Ensure --auth-no-challenge is honoured
* Test-auth-retcode.py: Ensure correct return code after 403 Forbidden
response.

2
testenv/Test--https.py
View File

@ -17,7 +17,7 @@ A_File = WgetFile ("File1", File1)
B_File = WgetFile ("File2", File2)
C_File = WgetFile ("File3", File3)
WGET_OPTIONS = "-d --no-check-certificate"
WGET_OPTIONS = "--no-check-certificate"
WGET_URLS = [["File1", "File2"]]
Files = [[A_File, B_File]]

2
testenv/Test--spider-r.py
View File

@ -74,7 +74,7 @@ Request_List = [
]
]
WGET_OPTIONS = "-d --spider -r"
WGET_OPTIONS = "--spider -r"
WGET_URLS = [[""]]
Files = [[index_html, secondpage_html, thirdpage_html, dummy_txt]]

2
testenv/Test-Content-disposition-2.py
View File

@ -20,7 +20,7 @@ File2_rules = {
A_File = WgetFile ("HTTP.Teapot", File1)
B_File = WgetFile ("File2", File2, rules=File2_rules)
WGET_OPTIONS = "-d --content-disposition"
WGET_OPTIONS = "--content-disposition"
WGET_URLS = [["File2"]]
Files = [[B_File]]

2
testenv/Test-Content-disposition.py
View File

@ -25,7 +25,7 @@ File1_rules = {
}
A_File = WgetFile ("LOTR", File1, rules=File1_rules)
WGET_OPTIONS = "-d --content-disposition"
WGET_OPTIONS = "--content-disposition"
WGET_URLS = [["LOTR"]]
Files = [[A_File]]

2
testenv/Test-Head.py
View File

@ -13,7 +13,7 @@ File1 = "You shall not pass!"
A_File = WgetFile ("File1", File1)
WGET_OPTIONS = "-d --method=HEAD"
WGET_OPTIONS = "--method=HEAD"
WGET_URLS = [["File1"]]
Files = [[A_File]]

2
testenv/Test-O.py
View File

@ -13,7 +13,7 @@ File1 = "Test Contents."
A_File = WgetFile ("File1", File1)
WGET_OPTIONS = "-d -O NewFile.txt"
WGET_OPTIONS = "-O NewFile.txt"
WGET_URLS = [["File1"]]
Files = [[A_File]]

2
testenv/Test-Parallel-Proto.py
View File

@ -18,7 +18,7 @@ A_File = WgetFile ("File1", File1)
B_File = WgetFile ("File2", File2)
C_File = WgetFile ("File3", File3)
WGET_OPTIONS = "-d"
WGET_OPTIONS = ""
WGET_URLS = [["File1"], ["File2"]]
Files = [[A_File], [B_File]]

2
testenv/Test-Post.py
View File

@ -17,7 +17,7 @@ TestMessage"""
A_File = WgetFile ("File1", File1)
WGET_OPTIONS = "-d --method=post --body-data=TestMessage"
WGET_OPTIONS = "--method=post --body-data=TestMessage"
WGET_URLS = [["File1"]]
Files = [[A_File]]

2
testenv/Test-Proto.py
View File

@ -39,7 +39,7 @@ A_File = WgetFile ("File1", File1, rules=File1_rules)
B_File = WgetFile ("File2", File2, rules=File2_rules)
C_File = WgetFile ("File3", File3)
WGET_OPTIONS = "-d --content-disposition --user=Sauron --password=TheEye"
WGET_OPTIONS = "--content-disposition --user=Sauron --password=TheEye"
WGET_URLS = [["File1", "File2"]]
Servers = [HTTP]

2
testenv/Test-auth-basic-fail.py
View File

@ -20,7 +20,7 @@ File1_rules = {
}
A_File = WgetFile ("File1", File1, rules=File1_rules)
WGET_OPTIONS = "-d --user=Sauron --password=Eye"
WGET_OPTIONS = "--user=Sauron --password=Eye"
WGET_URLS = [["File1"]]
Files = [[A_File]]

2
testenv/Test-auth-basic.py
View File

@ -28,7 +28,7 @@ File2_rules = {
A_File = WgetFile ("File1", File1, rules=File1_rules)
B_File = WgetFile ("File2", File2, rules=File2_rules)
WGET_OPTIONS = "-d --user=Sauron --password=TheEye"
WGET_OPTIONS = "--user=Sauron --password=TheEye"
WGET_URLS = [["File1", "File2"]]
Files = [[A_File, B_File]]

2
testenv/Test-auth-both.py
View File

@ -46,7 +46,7 @@ A_File = WgetFile ("File1", File1, rules=File1_rules)
B_File = WgetFile ("File2", File2, rules=File2_rules)
C_File = WgetFile ("File3", File3, rules=File3_rules)
WGET_OPTIONS = "-d --user=Sauron --password=TheEye"
WGET_OPTIONS = "--user=Sauron --password=TheEye"
WGET_URLS = [["File1", "File2", "File3"]]
Files = [[A_File, B_File, C_File]]

2
testenv/Test-auth-digest.py
View File

@ -19,7 +19,7 @@ File1_rules = {
}
A_File = WgetFile ("File1", File1, rules=File1_rules)
WGET_OPTIONS = "-d --user=Pacman --password=Omnomnom"
WGET_OPTIONS = "--user=Pacman --password=Omnomnom"
WGET_URLS = [["File1"]]
Files = [[A_File]]

2
testenv/Test-auth-no-challenge-url.py
View File

@ -23,7 +23,7 @@ File1_rules = {
}
A_File = WgetFile ("File1", File1, rules=File1_rules)
WGET_OPTIONS = "-d --auth-no-challenge http://Pacman:Omnomnom@localhost:{{port}}/File1"
WGET_OPTIONS = "--auth-no-challenge http://Pacman:Omnomnom@localhost:{{port}}/File1"
WGET_URLS = [[]]
Files = [[A_File]]

2
testenv/Test-auth-no-challenge.py
View File

@ -23,7 +23,7 @@ File1_rules = {
}
A_File = WgetFile ("File1", File1, rules=File1_rules)
WGET_OPTIONS = "-d --auth-no-challenge --user=Pacman --password=Omnomnom"
WGET_OPTIONS = "--auth-no-challenge --user=Pacman --password=Omnomnom"
WGET_URLS = [["File1"]]
Files = [[A_File]]

2
testenv/Test-auth-retcode.py
View File

@ -19,7 +19,7 @@ File1_rules = {
A_File = WgetFile ("File1", File1, rules=File1_rules)
WGET_OPTIONS = "-d"
WGET_OPTIONS = ""
WGET_URLS = [["File1"]]
Files = [[A_File]]

2
testenv/Test-auth-with-content-disposition.py
View File

@ -23,7 +23,7 @@ File1_rules = {
}
A_File = WgetFile ("File1", File1, rules=File1_rules)
WGET_OPTIONS = "-d --user=Pacman --password=Omnomnom --content-disposition"
WGET_OPTIONS = "--user=Pacman --password=Omnomnom --content-disposition"
WGET_URLS = [["File1"]]
Files = [[A_File]]

2
testenv/Test-c-full.py
View File

@ -20,7 +20,7 @@ D_File = WgetFile ("File2", File2)
E_File = WgetFile ("File3", File1)
WGET_OPTIONS = "-d -c"
WGET_OPTIONS = "-c"
WGET_URLS = [["File1", "File2", "File3"]]
Files = [[A_File, C_File, E_File]]

2
testenv/Test-cookie-401.py
View File

@ -28,7 +28,7 @@ File2_rules = {
A_File = WgetFile ("File1", File1, rules=File1_rules)
B_File = WgetFile ("File2", File2, rules=File2_rules)
WGET_OPTIONS = "-d"
WGET_OPTIONS = ""
WGET_URLS = [["File1", "File2"]]
Files = [[A_File, B_File]]

2
testenv/Test-cookie-domain-mismatch.py
View File

@ -26,7 +26,7 @@ File2_rules = {
A_File = WgetFile ("File1", File1, rules=File1_rules)
B_File = WgetFile ("File2", File2, rules=File2_rules)
WGET_OPTIONS = "-d"
WGET_OPTIONS = ""
WGET_URLS = [["File1", "File2"]]
Files = [[A_File, B_File]]

2
testenv/Test-cookie-expires.py
View File

@ -49,7 +49,7 @@ B_File = WgetFile ("File2", File2, rules=File2_rules)
C_File = WgetFile ("File3", File3, rules=File3_rules)
D_File = WgetFile ("File4", File4, rules=File4_rules)
WGET_OPTIONS = "-d"
WGET_OPTIONS = ""
WGET_URLS = [["File1", "File2", "File3", "File4"]]
Files = [[A_File, B_File, C_File, D_File]]

2
testenv/Test-cookie.py
View File

@ -26,7 +26,7 @@ File2_rules = {
A_File = WgetFile ("File1", File1, rules=File1_rules)
B_File = WgetFile ("File2", File2, rules=File2_rules)
WGET_OPTIONS = "-d"
WGET_OPTIONS = ""
WGET_URLS = [["File1", "File2"]]
Files = [[A_File, B_File]]

1
testenv/conf/__init__.py
View File

@ -3,6 +3,7 @@ import os
# this file implements the mechanism of conf class auto-registration,
# don't modify this file if you have no idea what you're doing
def gen_hook():
hook_table = {}

13
testenv/conf/authentication.py
View File

@ -1,5 +1,18 @@
from conf import rule
""" Rule: Authentication
This file defines an authentication rule which when applied to any file will
cause the server to prompt the client for the required authentication details
before serving it.
auth_type must be either of: Basic, Digest, Both or Both-inline
When auth_type is Basic or Digest, the server asks for the respective
authentication in its response. When auth_type is Both, the server sends two
Authenticate headers, one requesting Basic and the other requesting Digest
authentication. If auth_type is Both-inline, the server sends only one
Authenticate header, but lists both Basic and Digest as supported mechanisms in
that.
"""
@rule()
class Authentication:

5
testenv/conf/expect_header.py
View File

@ -1,5 +1,10 @@
from conf import rule
""" Rule: ExpectHeader
This rule defines a dictionary of headers and their value which the server
should expect in each request for the file to which the rule was applied.
"""
@rule()
class ExpectHeader:

11
testenv/conf/expected_files.py
View File

@ -4,6 +4,15 @@ import sys
from conf import hook
from exc.test_failed import TestFailed
""" Post-Test Hook: ExpectedFiles
This is a Post-Test hook that checks the test directory for the files it
contains. A dictionary object is passed to it, which contains a mapping of
filenames and contents of all the files that the directory is expected to
contain.
Raises a TestFailed exception if the expected files are not found or if extra
files are found, else returns gracefully.
"""
@hook()
class ExpectedFiles:
@ -34,7 +43,7 @@ class ExpectedFiles:
fromfile='Actual',
tofile='Expected'):
print(line, file=sys.stderr)
raise TestFailed('Contents of %s do not match.' % file.name)
raise TestFailed('Contents of %s do not match' % file.name)
else:
raise TestFailed('Expected file %s not found.' % file.name)
if local_fs:

8
testenv/conf/expected_ret_code.py
View File

@ -1,6 +1,14 @@
from exc.test_failed import TestFailed
from conf import hook
""" Post-Test Hook: ExpectedRetCode
This is a post-test hook which checks if the exit code of the Wget instance
under test is the same as that expected. As a result, this is a very important
post test hook which is checked in all the tests.
Returns a TestFailed exception if the return code does not match the expected
value. Else returns gracefully.
"""
@hook(alias='ExpectedRetcode')
class ExpectedRetCode:

9
testenv/conf/files_crawled.py
View File

@ -2,6 +2,15 @@ from misc.colour_terminal import print_red
from conf import hook
from exc.test_failed import TestFailed
""" Post-Test Hook: FilesCrawled
This is a post test hook that is invoked in tests that check wget's behaviour
in recursive mode. It expects an ordered list of the request lines that Wget
must send to the server. If the requests received by the server do not match
the provided list, IN THE GIVEN ORDER, then it raises a TestFailed exception.
Such a test can be used to check the implementation of the recursion algorithm
in Wget too.
"""
@hook()
class FilesCrawled:

9
testenv/conf/hook_sample.py
View File

@ -1,7 +1,12 @@
from exc.test_failed import TestFailed
from conf import hook
# this file is a hook example
""" Hook: SampleHook
This a sample file for how a new hook should be defined.
Any errors should always be reported by raising a TestFailed exception instead
of returning a true or false value.
"""
@hook(alias='SampleHookAlias')
class SampleHook:
@ -12,4 +17,6 @@ class SampleHook:
def __call__(self, test_obj):
# implement hook here
# if you need the test case instance, refer to test_obj
if False:
raise TestFailed ("Reason")
pass

6
testenv/conf/local_files.py
View File

@ -1,5 +1,11 @@
from conf import hook
""" Pre-Test Hook: LocalFiles
This is a pre-test hook used to generate the specific environment before a test
is run. The LocalFiles hook creates the files which should exist on disk before
invoking Wget.
"""
@hook()
class LocalFiles:

6
testenv/conf/reject_header.py
View File

@ -1,5 +1,11 @@
from conf import rule
""" Rule: RejectHeader
This is a server side rule which expects a dictionary object of Headers and
their values which should be blacklisted by the server for a particular file's
requests.
"""
@rule()
class RejectHeader:

4
testenv/conf/response.py
View File

@ -1,5 +1,9 @@
from conf import rule
""" Rule: Response
When this rule is set against a certain file, the server will unconditionally
respond to any request for the said file with the provided response code. """
@rule()
class Response:

5
testenv/conf/send_header.py
View File

@ -1,5 +1,10 @@
from conf import rule
""" Rule: SendHeader
Have the server send custom headers when responding to a request for the file
this rule is applied to. The header_obj object is expected to be dictionary
mapping headers to their contents. """
@rule()
class SendHeader:

11
testenv/conf/server_files.py
View File

@ -1,5 +1,16 @@
from conf import hook
""" Pre-Test Hook: ServerFiles
This hook is used to define a set of files on the server's virtual filesystem.
server_files is expected to be dictionary that maps filenames to their
contents. In the future, this can be used to add additional metadat to the
files using the WgetFile class too.
This hook also does some additional processing on the contents of the file. Any
text between {{and}} is replaced by the contents of a class variable of the
same name. This is useful in creating files that contain an absolute link to
another file on the same server. """
@hook()
class ServerFiles:

4
testenv/conf/urls.py
View File

@ -1,5 +1,9 @@
from conf import hook
""" Pre-Test Hook: URLS
This hook is used to define the paths of the files on the test server that wget
will send a request for. """
@hook(alias='Urls')
class URLs:

5
testenv/conf/wget_commands.py
View File

@ -1,5 +1,10 @@
from conf import hook
""" Pre-Test Hook: WgetCommands
This hook is used to specify the test specific switches that must be passed to
wget on invokation. Default switches are hard coded in the test suite itself.
"""
@hook()
class WgetCommands:

7
testenv/exc/server_error.py Normal file
View File

@ -0,0 +1,7 @@
class ServerError (Exception):
""" A custom exception which is raised by the test servers. Often used to
handle control flow. """
def __init__ (self, err_message):
self.err_message = err_message

45
testenv/misc/colour_terminal.py
View File

@ -2,24 +2,39 @@ from functools import partial
import platform
from os import getenv
""" This module allows printing coloured output to the terminal when running a
Wget Test under certain conditions.
The output is coloured only on Linux systems. This is because coloured output
in the terminal on Windows requires too much effort for what is simply a
convenience. This might work on OSX terminals, but without a confirmation, it
remains unsupported.
Another important aspect is that the coloured output is printed only if the
environment variable MAKE_CHECK is not set. This variable is set when running
the test suite through, `make check`. In that case, the output is not only
printed to the terminal but also copied to a log file where the ANSI escape
codes on;y add clutter. """
T_COLORS = {
'PURPLE' : '\033[95m',
'BLUE' : '\033[94m',
'GREEN' : '\033[92m',
'YELLOW' : '\033[93m',
'RED' : '\033[91m',
'ENDC' : '\033[0m'
'PURPLE' : '\033[95m',
'BLUE' : '\033[94m',
'GREEN' : '\033[92m',
'YELLOW' : '\033[93m',
'RED' : '\033[91m',
'ENDC' : '\033[0m'
}
def printer (color, string):
if platform.system () == 'Linux':
if getenv ("MAKE_CHECK", "False") == "True":
print (string)
else:
print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC'))
else:
print (string)
def printer (color, string):
if platform.system () == 'Linux':
if getenv ("MAKE_CHECK", "False") == "True":
print (string)
else:
print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC'))
else:
print (string)
print_blue = partial(printer, 'BLUE')
@ -28,4 +43,4 @@ print_green = partial(printer, 'GREEN')
print_purple = partial(printer, 'PURPLE')
print_yellow = partial(printer, 'YELLOW')
# vim: set ts=8 sw=3 tw=0 et :
# vim: set ts=8 sw=3 tw=80 et :

190
testenv/server/http/http_server.py
View File

@ -1,4 +1,5 @@
from http.server import HTTPServer, BaseHTTPRequestHandler
from exc.server_error import ServerError
from socketserver import BaseServer
from posixpath import basename, splitext
from base64 import b64encode
@ -11,20 +12,12 @@ import ssl
import os
class InvalidRangeHeader (Exception):
""" Create an Exception for handling of invalid Range Headers. """
# TODO: Eliminate this exception and use only ServerError
def __init__ (self, err_message):
self.err_message = err_message
class ServerError (Exception):
def __init__ (self, err_message):
self.err_message = err_message
class StoppableHTTPServer (HTTPServer):
""" This class extends the HTTPServer class from default http.server library
in Python 3. The StoppableHTTPServer class is capable of starting an HTTP
server that serves a virtual set of files made by the WgetFile class and
has most of its properties configurable through the server_conf()
method. """
request_headers = list ()
@ -35,45 +28,45 @@ class StoppableHTTPServer (HTTPServer):
self.server_configs = conf_dict
self.fileSys = filelist
def server_sett (self, settings):
for settings_key in settings:
setattr (self.RequestHandlerClass, settings_key, settings[settings_key])
def get_req_headers (self):
return self.request_headers
class HTTPSServer (StoppableHTTPServer):
""" The HTTPSServer class extends the StoppableHTTPServer class with
additional support for secure connections through SSL. """
def __init__ (self, address, handler):
BaseServer.__init__ (self, address, handler)
print (os.getcwd())
CERTFILE = os.path.abspath (os.path.join ('..', 'certs', 'wget-cert.pem'))
print (CERTFILE)
fop = open (CERTFILE)
print (fop.readline())
self.socket = ssl.wrap_socket (
sock = socket.socket (self.address_family, self.socket_type),
ssl_version = ssl.PROTOCOL_TLSv1,
certfile = CERTFILE,
server_side = True
)
self.server_bind ()
self.server_activate ()
def __init__ (self, address, handler):
BaseServer.__init__ (self, address, handler)
print (os.getcwd())
CERTFILE = os.path.abspath(os.path.join('..', 'certs', 'wget-cert.pem'))
print (CERTFILE)
fop = open (CERTFILE)
print (fop.readline())
self.socket = ssl.wrap_socket (
sock = socket.socket (self.address_family, self.socket_type),
ssl_version = ssl.PROTOCOL_TLSv1,
certfile = CERTFILE,
server_side = True
)
self.server_bind()
self.server_activate()
class WgetHTTPRequestHandler (BaseHTTPRequestHandler):
""" Define methods for handling Test Checks. """
class _Handler (BaseHTTPRequestHandler):
""" This is a private class which tells the server *HOW* to handle each
request. For each HTTP Request Command that the server should be capable of
responding to, there must exist a do_REQUESTNAME() method which details the
steps in which such requests should be processed. The rest of the methods
in this class are auxilliary methods created to help in processing certain
requests. """
def get_rule_list (self, name):
r_list = self.rules.get (name) if name in self.rules else None
return r_list
class _Handler (WgetHTTPRequestHandler):
""" Define Handler Methods for different Requests. """
InvalidRangeHeader = InvalidRangeHeader
# The defailt protocol version of the server we run is HTTP/1.1 not
# HTTP/1.0 which is the default with the http.server module.
protocol_version = 'HTTP/1.1'
""" Define functions for various HTTP Requests. """
@ -82,6 +75,11 @@ class _Handler (WgetHTTPRequestHandler):
self.send_head ("HEAD")
def do_GET (self):
""" Process HTTP GET requests. This is the same as processing HEAD
requests and then actually transmitting the data to the client. If
send_head() does not specify any "start" offset, we send the complete
data, else transmit only partial data. """
content, start = self.send_head ("GET")
if content:
if start is None:
@ -90,11 +88,26 @@ class _Handler (WgetHTTPRequestHandler):
self.wfile.write (content.encode ('utf-8')[start:])
def do_POST (self):
""" According to RFC 7231 sec 4.3.3, if the resource requested in a POST
request does not exist on the server, the first POST request should
create that resource. PUT requests are otherwise used to create a
resource. Hence, we call the handle for processing PUT requests if the
resource requested does not already exist.
Currently, when the server recieves a POST request for a resource, we
simply append the body data to the existing file and return the new
file to the client. If the file does not exist, a new file is created
using the contents of the request body. """
path = self.path[1:]
self.rules = self.server.server_configs.get (path)
if not self.custom_response ():
return (None, None)
if path in self.server.fileSys:
self.rules = self.server.server_configs.get (path)
if not self.rules:
self.rules = dict ()
if not self.custom_response ():
return (None, None)
body_data = self.get_body_data ()
self.send_response (200)
self.send_header ("Content-type", "text/plain")
@ -102,6 +115,7 @@ class _Handler (WgetHTTPRequestHandler):
total_length = len (content)
self.server.fileSys[path] = content
self.send_header ("Content-Length", total_length)
self.send_header ("Location", self.path)
self.finish_headers ()
try:
self.wfile.write (content.encode ('utf-8'))
@ -115,7 +129,6 @@ class _Handler (WgetHTTPRequestHandler):
self.rules = self.server.server_configs.get (path)
if not self.custom_response ():
return (None, None)
self.server.fileSys.pop (path, None)
self.send_put (path)
""" End of HTTP Request Method Handlers. """
@ -126,12 +139,12 @@ class _Handler (WgetHTTPRequestHandler):
if header_line is None:
return None
if not header_line.startswith ("bytes="):
raise InvalidRangeHeader ("Cannot parse header Range: %s" %
(header_line))
raise ServerError ("Cannot parse header Range: %s" %
(header_line))
regex = re.match (r"^bytes=(\d*)\-$", header_line)
range_start = int (regex.group (1))
if range_start >= length:
raise InvalidRangeHeader ("Range Overflow")
raise ServerError ("Range Overflow")
return range_start
def get_body_data (self):
@ -141,23 +154,27 @@ class _Handler (WgetHTTPRequestHandler):
return body_data
def send_put (self, path):
if path in self.server.fileSys:
self.server.fileSys.pop (path, None)
self.send_response (204)
else:
self.rules = dict ()
self.send_response (201)
body_data = self.get_body_data ()
self.send_response (201)
self.server.fileSys[path] = body_data
self.send_header ("Content-type", "text/plain")
self.send_header ("Content-Length", len (body_data))
self.send_header ("Location", self.path)
self.finish_headers ()
try:
self.wfile.write (body_data.encode ('utf-8'))
except Exception:
pass
""" This empty method is called automatically when all the rules are
processed for a given request. However, send_header() should only be called
AFTER a response has been sent. But, at the moment of processing the rules,
the appropriate response has not yet been identified. As a result, we defer
the processing of this rule till later. Each do_* request handler MUST call
finish_headers() instead of end_headers(). The finish_headers() method
takes care of sending the appropriate headers before completing the
response. """
def SendHeader (self, header_obj):
pass
# headers_list = header_obj.headers
# for header_line in headers_list:
# print (header_line + " : " + headers_list[header_line])
# self.send_header (header_line, headers_list[header_line])
def send_cust_headers (self):
header_obj = self.get_rule_list ('SendHeader')
@ -195,11 +212,11 @@ class _Handler (WgetHTTPRequestHandler):
if auth_type == "Basic":
challenge_str = 'Basic realm="Wget-Test"'
elif auth_type == "Digest" or auth_type == "Both_inline":
self.nonce = md5 (str (random ()).encode ('utf-8')).hexdigest ()
self.opaque = md5 (str (random ()).encode ('utf-8')).hexdigest ()
challenge_str = 'Digest realm="Test", nonce="%s", opaque="%s"' %(
self.nonce,
self.opaque)
self.nonce = md5 (str (random ()).encode ('utf-8')).hexdigest()
self.opaque = md5 (str (random ()).encode ('utf-8')).hexdigest()
challenge_str = 'Digest realm="Test", nonce="%s", opaque="%s"' % (
self.nonce,
self.opaque)
challenge_str += ', qop="auth"'
if auth_type == "Both_inline":
challenge_str = 'Basic realm="Wget-Test", ' + challenge_str
@ -218,9 +235,9 @@ class _Handler (WgetHTTPRequestHandler):
n = len("Digest ")
auth_header = auth_header[n:].strip()
items = auth_header.split(", ")
key_values = [i.split("=", 1) for i in items]
key_values = [(k.strip(), v.strip().replace('"', '')) for k, v in key_values]
return dict(key_values)
keyvals = [i.split("=", 1) for i in items]
keyvals = [(k.strip(), v.strip().replace('"', '')) for k, v in keyvals]
return dict(keyvals)
def KD (self, secret, data):
return self.H (secret + ":" + data)
@ -237,10 +254,10 @@ class _Handler (WgetHTTPRequestHandler):
def check_response (self, params):
if "qop" in params:
data_str = params['nonce'] \
+ ":" + params['nc'] \
+ ":" + params['cnonce'] \
+ ":" + params['qop'] \
+ ":" + self.H (self.A2 (params))
+ ":" + params['nc'] \
+ ":" + params['cnonce'] \
+ ":" + params['qop'] \
+ ":" + self.H (self.A2 (params))
else:
data_str = params['nonce'] + ":" + self.H (self.A2 (params))
resp = self.KD (self.H (self.A1 ()), data_str)
@ -256,11 +273,12 @@ class _Handler (WgetHTTPRequestHandler):
params = self.parse_auth_header (auth_header)
pass_auth = True
if self.user != params['username'] or \
self.nonce != params['nonce'] or self.opaque != params['opaque']:
self.nonce != params['nonce'] or \
self.opaque != params['opaque']:
pass_auth = False
req_attribs = ['username', 'realm', 'nonce', 'uri', 'response']
for attrib in req_attribs:
if not attrib in params:
if attrib not in params:
pass_auth = False
if not self.check_response (params):
pass_auth = False
@ -326,19 +344,6 @@ class _Handler (WgetHTTPRequestHandler):
self.finish_headers ()
raise ServerError ("Header " + header_line + " not found")
def expect_headers (self):
""" This is modified code to handle a few changes. Should be removed ASAP """
exp_headers_obj = self.get_rule_list ('ExpectHeader')
if exp_headers_obj:
exp_headers = exp_headers_obj.headers
for header_line in exp_headers:
header_re = self.headers.get (header_line)
if header_re is None or header_re != exp_headers[header_line]:
self.send_error (400, 'Expected Header not Found')
self.end_headers ()
return False
return True
def RejectHeader (self, header_obj):
rej_headers = header_obj.headers
for header_line in rej_headers:
@ -400,7 +405,7 @@ class _Handler (WgetHTTPRequestHandler):
try:
self.range_begin = self.parse_range_header (
self.headers.get ("Range"), content_length)
except InvalidRangeHeader as ae:
except ServerError as ae:
# self.log_error("%s", ae.err_message)
if ae.err_message == "Range Overflow":
self.send_response (416)
@ -431,9 +436,9 @@ class _Handler (WgetHTTPRequestHandler):
base_name = basename ("/" + path)
name, ext = splitext (base_name)
extension_map = {
".txt" : "text/plain",
".css" : "text/css",
".html" : "text/html"
".txt" : "text/plain",
".css" : "text/css",
".html" : "text/html"
}
if ext in extension_map:
return extension_map[ext]
@ -444,6 +449,7 @@ class _Handler (WgetHTTPRequestHandler):
class HTTPd (threading.Thread):
server_class = StoppableHTTPServer
handler = _Handler
def __init__ (self, addr=None):
threading.Thread.__init__ (self)
if addr is None:
@ -452,16 +458,14 @@ class HTTPd (threading.Thread):
self.server_address = self.server_inst.socket.getsockname()[:2]
def run (self):
self.server_inst.serve_forever ()
self.server_inst.serve_forever ()
def server_conf (self, file_list, server_rules):
self.server_inst.server_conf (file_list, server_rules)
def server_sett (self, settings):
self.server_inst.server_sett (settings)
class HTTPSd (HTTPd):
server_class = HTTPSServer
server_class = HTTPSServer
# vim: set ts=4 sts=4 sw=4 tw=80 et :

9
testenv/test/base_test.py
View File

@ -28,9 +28,9 @@ class BaseTest:
Attributes should not be defined outside __init__.
"""
self.name = name
self.pre_configs = pre_hook or {} # if pre_hook == None, then
# {} (an empty dict object) is
# passed to self.pre_configs
self.pre_configs = pre_hook or {} # if pre_hook == None, then
# {} (an empty dict object) is
# passed to self.pre_configs
self.test_params = test_params or {}
self.post_configs = post_hook or {}
self.protocols = protocols
@ -98,12 +98,13 @@ class BaseTest:
test_path = os.path.abspath(".")
wget_path = os.path.abspath(os.path.join(test_path,
"..", '..', 'src', "wget"))
wget_options = '--debug --no-config %s' % self.wget_options
if os.getenv("VALGRIND_TESTS"):
valgrind_test = "valgrind --error-exitcode=301 --leak-check=full"
else:
valgrind_test = ""
cmd_line = '%s %s %s ' % (valgrind_test, wget_path, self.wget_options)
cmd_line = '%s %s %s ' % (valgrind_test, wget_path, wget_options)
for protocol, urls, domain in zip(self.protocols,
self.urls,
self.domains):

7
testenv/test/http_test.py
View File

@ -7,9 +7,10 @@ class HTTPTest(BaseTest):
""" Class for HTTP Tests. """
# Temp Notes: It is expected that when pre-hook functions are executed, only an empty test-dir exists.
# pre-hook functions are executed just prior to the call to Wget is made.
# post-hook functions will be executed immediately after the call to Wget returns.
# Temp Notes: It is expected that when pre-hook functions are executed,
# only an empty test-dir exists. pre-hook functions are executed just prior
# to the call to Wget is made. post-hook functions will be executed
# immediately after the call to Wget returns.
def __init__(self,
name="Unnamed Test",