diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..2b3c5967
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,91 @@
+# Project Root
+/.sc-start-sc_bindtextdomain
+/.sc-start-sc_prohibit_HAVE_MBRTOWC
+/.version
+/GNUmakefile
+/INSTALL
+/Makefile
+/Makefile.in
+/aclocal.m4
+/autom4te.cache/
+/config.cache
+/config.log
+/config.status
+/configure
+/lib/
+/maint.mk
+# Wildcard Ignores
+*~
+po/*.gmo*
+po/*.po*
+src/*.o
+testenv/*.log
+testenv/*.trs
+# build-aux/
+build-aux/.gitignore
+build-aux/compile
+build-aux/config.guess
+build-aux/config.sub
+build-aux/depcomp
+build-aux/install-sh
+build-aux/mdate-sh
+build-aux/missing
+build-aux/snippet/
+build-aux/test-driver
+build-aux/texinfo.tex
+# doc/
+doc/Makefile
+doc/Makefile.in
+doc/sample.wgetrc.munged_for_texi_inclusion
+doc/stamp-vti
+doc/version.texi
+doc/wget.1
+doc/wget.info
+doc/wget.pod
+# m4/
+m4/.gitignore
+m4/gnulib-cache.m4
+# po/
+po/.gitignore
+po/.reference/
+po/LINGUAS
+po/Makefile
+po/Makefile.in
+po/Makevars
+po/POTFILES
+po/wget.pot
+# src/
+src/.deps/
+src/Makefile
+src/Makefile.in
+src/build_info.c
+src/config.h
+src/config.h.in
+src/cscope.out
+src/css.c
+src/css_.c
+src/libunittest.a
+src/stamp-h1
+src/tags
+src/version.c
+src/wget
+# testenv/
+testenv/conf/__pycache__/
+testenv/exc/__pycache__/
+testenv/misc/__pycache__/
+testenv/server/__pycache__/
+testenv/server/http/__pycache__/
+testenv/test/__pycache__/
+testenv/Makefile
+testenv/Makefile.in
+# tests/
+tests/Makefile
+tests/Makefile.in
+tests/WgetTest.pm
+tests/unit-tests
+# util/
+util/Makefile
+util/Makefile.in
+# others
+/tags
+/cscope.out
diff --git a/ChangeLog b/ChangeLog
index 40365935..a07c1984 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,20 @@
+2014-07-25 Darshit Shah <darnir@gmail.com>
+
+ * .gitignore: Add a gitignore file for the project.
+
+2013-08-31 Darshit Shah <darnir@gmail.com>
+
+ * configure.ac: Add testenv/Makefile to AC_CONFIG_FILES.
+ * Makefile.am: Add testenv to SUBDIRS
+
+2014-07-22 Darshit Shah <darnir@gmail.com>
+
+ * configure.ac: Fix broken code for detecting libpsl
+
+2014-07-21 Darshit Shah <darnir@gmail.com>
+
+ * configure.ac: Fix check for Libpsl
+
2014-06-28 Giuseppe Scrivano <gscrivan@redhat.com>
* cfg.mk (local-checks-to-skip): Remove some checks.
diff --git a/configure.ac b/configure.ac
index 152b22f5..0f83fee7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -61,9 +61,15 @@ dnl
dnl Process features.
dnl
+ENABLE_PSL=no
AC_ARG_WITH(libpsl,
AS_HELP_STRING([--without-libpsl],
- [disable support for libpsl cookie checking.]))
+ [disable support for libpsl cookie checking.]),
+ [],
+ [AC_SEARCH_LIBS(psl_builtin, psl,
+ [ENABLE_PSL=yes; AC_DEFINE([HAVE_LIBPSL], [1], [PSL Support Enabled])],
+ [AC_MSG_WARN(*** libpsl not found. Falling back to Wget builtin cookie checking.)])
+ ])
AC_ARG_WITH(ssl,
[[ --without-ssl disable SSL autodetection
@@ -250,11 +256,6 @@ dnl
dnl Checks for libraries.
dnl
-AS_IF([test x"$with_libpsl" != xno], [
- with_libpsl=yes
- AC_CHECK_LIB([psl], [psl_builtin])
-])
-
AS_IF([test x"$with_zlib" != xno], [
with_zlib=yes
AC_CHECK_LIB(z, compress)
@@ -675,7 +676,7 @@ AC_MSG_NOTICE([Summary of build options:
Libs: $LIBS
SSL: $with_ssl
Zlib: $with_zlib
- PSL: $with_libpsl
+ PSL: $ENABLE_PSL
Digest: $ENABLE_DIGEST
NTLM: $ENABLE_NTLM
OPIE: $ENABLE_OPIE
diff --git a/doc/ChangeLog b/doc/ChangeLog
index f9967dad..d5e74961 100644
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -1,3 +1,7 @@
+2014-08-03 Giuseppe Scrivano <gscrivano@gnu.org>
+
+ * wget.texi (Download Options): Fix texinfo warning.
+
2014-06-11 Giuseppe Scrivano <gscrivan@redhat.com>
* Makefile.am: Remove @VAR@ with $FOO.
diff --git a/doc/wget.texi b/doc/wget.texi
index c487e1e1..9c2459c8 100644
--- a/doc/wget.texi
+++ b/doc/wget.texi
@@ -956,7 +956,8 @@ Don't use proxies, even if the appropriate @code{*_proxy} environment
variable is defined.
@c man end
-@xref{Proxies} for more information about the use of proxies with Wget.
+@xref{Proxies}, for more information about the use of proxies with
+Wget.
@c man begin OPTIONS
@cindex quota
@@ -1306,8 +1307,8 @@ really important, do not leave them lying in those files either---edit
the files and delete them after Wget has started the download.
@iftex
-@xref{Security Considerations} for more information about security issues
-with Wget.
+@xref{Security Considerations}, for more information about security
+issues with Wget.
@end iftex
@cindex Keep-Alive, turning off
@@ -1843,8 +1844,8 @@ really important, do not leave them lying in those files either---edit
the files and delete them after Wget has started the download.
@iftex
-@xref{Security Considerations} for more information about security issues
-with Wget.
+@xref{Security Considerations}, for more information about security
+issues with Wget.
@end iftex
@cindex .listing files, removing
diff --git a/src/ChangeLog b/src/ChangeLog
index 71aef6ec..df6deabf 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -2,6 +2,23 @@
* Makefile.am (wget_SOURCES): Remove space-tab indentation.
+2014-07-23 Darshit Shah <darnir@gmail.com>
+
+ * http.c (gethttp): Fix a memory leak when retrying authorization
+ (gethttp): Fix memory leak when trying to parse content disposition headers
+ (http_loop): Assigning a new value to *local)file without freeing the old
+ one causes a memory leak
+ (http_loop): Free the HTTP message and error strings before continuing loop
+
+2014-07-21 Daniel Stenberg <daniel@haxx.se>
+
+ * main.c (print_help): HTTP Method is a part of the Request not Header
+
+2014-07-21 Darshit Shah <darnir@gmail.com>
+
+ * cookies.c (check_domain_match): Fix a potential memory leak when checking
+ cookie domain names
+
2014-07-07 Tomas Hozza <thozza@redhat.com>
* iri.c (locale_to_utf8): Fix checking of iconv_open return code.
diff --git a/src/cookies.c b/src/cookies.c
index 76301acc..bf872a88 100644
--- a/src/cookies.c
+++ b/src/cookies.c
@@ -546,9 +546,12 @@ check_domain_match (const char *cookie_domain, const char *host)
xfree (cookie_domain_lower);
xfree (host_lower);
- return true ? (is_acceptable == 1) : false;
+ return is_acceptable == 1;
no_psl:
+ /* Cleanup the PSL pointers first */
+ xfree (cookie_domain_lower);
+ xfree (host_lower);
#endif
/* For efficiency make some elementary checks first */
diff --git a/src/http.c b/src/http.c
index 61034607..fc79ecf5 100644
--- a/src/http.c
+++ b/src/http.c
@@ -2702,6 +2702,7 @@ read_header:
resp_free (resp);
xfree (head);
xfree (auth_stat);
+ xfree (hs->message);
goto retry_with_auth;
}
else
@@ -2762,6 +2763,8 @@ read_header:
local_file));
hs->local_file = url_file_name (u, local_file);
}
+
+ xfree_null (local_file);
}
/* TODO: perform this check only once. */
@@ -3726,6 +3729,8 @@ Remote file exists.\n\n"));
got_name = true;
*dt &= ~HEAD_ONLY;
count = 0; /* the retrieve count for HEAD is reset */
+ xfree_null (hstat.message);
+ xfree_null (hstat.error);
continue;
} /* send_head_first */
} /* !got_head */
@@ -3873,7 +3878,10 @@ Remote file exists.\n\n"));
exit:
if (ret == RETROK && local_file && !(*local_file))
- *local_file = xstrdup (hstat.local_file);
+ {
+ xfree_null (*local_file);
+ *local_file = xstrdup (hstat.local_file);
+ }
free_hstat (&hstat);
return ret;
diff --git a/src/main.c b/src/main.c
index 4e6043e4..53fea6f8 100644
--- a/src/main.c
+++ b/src/main.c
@@ -636,7 +636,7 @@ HTTP options:\n"),
N_("\
--post-file=FILE use the POST method; send contents of FILE.\n"),
N_("\
- --method=HTTPMethod use method \"HTTPMethod\" in the header.\n"),
+ --method=HTTPMethod use method \"HTTPMethod\" in the request.\n"),
N_("\
--body-data=STRING Send STRING as data. --method MUST be set.\n"),
N_("\
diff --git a/testenv/ChangeLog b/testenv/ChangeLog
index 97d2266c..f9a45ab5 100644
--- a/testenv/ChangeLog
+++ b/testenv/ChangeLog
@@ -454,3 +454,505 @@
* Test-auth-no-challenge.py: Ensure --auth-no-challenge is honoured
* Test-auth-retcode.py: Ensure correct return code after 403 Forbidden
response.
+2014-08-08 Darshit Shah <darnir@gmail.com>
+
+ * conf/__init__.py: Add extra newline according to PEP8
+ * conf/{authentication,expect_header,expected_files,expected_ret_code,
+ files_crawled,hook_sample,local_files,reject_header,response,send_header,
+ server_files,urls,wget_commands}.py: Add docstrings explaining the conf file
+ and how it should be used
+ * server/http/http_server (InvalidRangeHeader): Clear TODO and eliminate
+ this exception. Use ServerError for all such purposes.
+ (_Handler): Remove reference to InvalidRangeHeader
+ (_handler.parse_range_header): User ServerError instead of InvalidRangeHeader
+ (_Handler.do_GET): Add docstring
+ (_Handler.do_POST): Add docstring. Also create an empty dict for rules if
+ no rules are supplied. Send the Location header as suggested in RFC 7231
+ (_Handler.do_PUT): Don't pop the server file already. Push it to later in ..
+ (_Handler.send_put): .. Here. If the file exists respond with a 204 No
+ Content message and pop the file for replacement. Do not send the
+ Content-Length, Content-Type headers since PUT requests should not respond
+ with data.
+ (_Handler.parse_auth_header): Fit line within 80 chars
+ (_Handler.check_response): Better visual indent
+ (_Handler.authorize_digest): Better visual indent.
+ (_Handler.expect_headers): Remove unused function
+ (_Handler.guess_type): Fix indentation
+ (HTTPd): Add newline according to PEP8 guidelines
+ (HTTPSd): Fix indentation
+ (StoppableHTTPServer): Add docstring
+ (HTTPSServer): Fix indentation
+ (WgetHTTPRequestHandler): Merge class into _handler.
+ (_Handler): Add docstring
+ (_Handler.parse_range_header): Fix indentation
+ (ServerError): Split exception into separate file ...
+ * exc/server_error.py: ... Here
+ * misc/colour_terminal.py: Add docstring, fix indentation
+ * test/base_test.py: Fix visual indent
+ * test/http_test.py: Fit within 80 char lines
+
+2014-08-04 Darshit Shah <darnir@gmail.com>
+
+ * conf/server_conf.py: Delete file. Server configuration is now done via the
+ server_conf() method.
+ * server/http/http_server.py (StppableHTTPServer.server_sett): Delete
+ method required by the above hook
+ (HTTPd.server_sett): Same
+
+2014-07-26 Darshit Shah <darnir@gmail.com>
+
+ * Test-*.py: Remove the '-d' switch from WGET_OPTIONS.
+ * test/base_test (BaseTest.gen_cmd_line): Add --debug and --no-config to the
+ list of switches passed to wget unconditionally.
+
+2014-07-23 Darshit Shah <darnir@gmail.com>
+
+ * test/base_test.py (BaseTest.gen_cmd_line): Add support for running all
+ tests through valgrind if the relevant environment variable is set
+ * conf/expected_ret_code (ExpectedRetCode.__call__): Valgrind returns error
+ code 45 when it detects a memory leak.
+ * Readme: Update with details about valgrind tests
+
+2014-07-22 Darshit Shah <darnir@gmail.com>
+
+ * (README): Remove old TODO and document SERVER_WAIT variable
+
+2014-06-22 Darshit Shah <darnir@gmail.com>
+
+ * (conf.files_crawled): diff is a set object and needs explicit str
+ conversion.
+
+2014-03-13 Zihang Chen <chsc4698@gmail.com>
+
+ * base_test.py:
+ (CommonMethods): Rename to BaseTest.
+ (BaseTest): Implement __init__ method where the class-wide variables are
+ initialized. Also variable names like `xxx_list` is renamed to its plural
+ form, e.g. `server_list` => `servers`.
+ (BaseTest.init_test_env): Remove name argument due to its unnecessarity.
+ (BaseTest.get_test_dir): Because the path of the test directory is needed
+ in multiple methods, this method is implemented.
+ (BaseTest.get_domain_addr): Rewrite the return statement utilizing str
+ formatting (which is more Pythonic).
+ (BaseTest.get_cmd_line): Rename to gen_cmd_line. Change the variables with
+ capitcal characters to lower ones. Also, the nested for loop is rewritten
+ to a plain loop using the zip function.
+ (BaseTest.__gen_local_filesys): Rename to gen_local_fs_snapshot. Move to
+ ExpectedFiles in conf/expected_files.py and is marked as a static
+ method. Refactor to a less verbose implementation.
+ (BaseTest._check_downloaded_files): Rename to __call__ to agree with the
+ invocation in test case classes. Move to ExpectedFiles in
+ conf/expected_files.py.
+ (BaseTest.get_server_rules): Refactor to a more Pythonic form utilizing
+ dict.items() and is marked static.
+ (BaseTest.stop_server): (new method) an abstract method which should stop
+ the currently using servers.
+ (BaseTest.instantiate_server_by): (new method) an abstract method which
+ should instantiate a server instance according to the given argument.
+ (BaseTest.__enter__): (new method) method which initialize the context
+ manager
+ (BaseTest.__exit__): (new method) method that finilize the context manager
+ and deal with the exceptions during the execution of the with statement,
+ subclasses can override this method for extensibility
+ * http_test.py:
+ (HTTPTest.__init__): Add call to super.__init__. Default values of
+ pre_hook, test_params, post_hook are set to None to avoid a subtle bug of
+ Python. Argument servers is renamed to protocols.
+ (HTTPTest.Server_setup): Move to BaseTest and rename to server_setup.
+ Calls to pre_hook_call, call_test, post_hook_call are removed.
+ (HTTPTest.hook_call, pre_hook_call, call_test, post_hook_call): Move to
+ BaseTest for that both HTTP test cases and FTP test cases may use these
+ methods.
+ (HTTPTest.init_HTTP_Server, init_HTTPS_Server): Merge and rename to
+ instantiate_server_by to implement the abstract method in BaseTest.
+ (HTTPTest.stop_HTTP_Server): Rename to stop_server to implement the
+ abstract method in BaseTest. Also, pull out the part where remaining
+ requests are gathered into a new method request_remaining.
+ (BaseTest.act_retcode): Rename to ret_code because ExpectedRetCode is
+ moved out from BaseTest, so the name act_retcode is actually a bit
+ verbose.
+ * conf/expected_ret_code.py:
+ (ExpectedRetCode.__call__): Rewrite the str into a more readable form.
+ * conf/files_crawled.py:
+ (FilesCrawled.__call__): Refactor this method into a more Pythonic form
+ utilizing the zip function.
+ * conf/local_files.py:
+ (LocalFiles__call__): Rewrite this method with the recommended with
+ statement.
+ * conf/server_conf.py:
+ (ServerConf.__call__): Rewrite this method due to BaseTest.server_list is
+ renamed to BaseTest.servers.
+ * conf/server_files.py:
+ (ServerFiles.__call__): Refactor the nested for loop into a plain one
+ utilizing the zip function.
+ * conf/urls.py:
+ (URLs): Rename url_list to urls.
+ * conf/wget_commands.py:
+ (WgetCommands): Rename command_list to commands, rename test_obj.options
+ to test_obj.wget_options.
+ * Test--https.py, Test-Proto.py, Test-Parallel-Proto.py: Argument servers
+ is changed to protocols due to change in the signature of
+ HTTPTest.__init__.
+
+2014-03-13 Zihang Chen <chsc4698@gmail.com>
+
+ * test: (new package) package for test case classes
+ * WgetTest.py: Split into test/base_test.py and test/http_test.py.
+ * Test-*.py: Optimize the imports according to changes of WgetTest.py
+
+2014-03-13 Zihang Chen <chsc4698@gmail.com>
+
+ * server: (new package) package for the server classes
+ * server.http: (new package) package for HTTP server
+ * server.ftp: (new package) package for FTP server
+ * HTTPServer.py: Move to server/http/http_server.py. Also change the
+ CERTFILE to '../certs/wget-cert.pem'.
+ * FTPServer.py: Move to server/ftp/ftp_server.py.
+ * WgetTest.py: Optimize import respect to the server classes.
+
+2014-03-13 Zihang Chen <chsc4698@gmail.com>
+
+ * conf: (new package) package for rule classes and hook methods
+ * WgetTest.py:
+ (CommonMethods.Authentication): Move to conf/authentication.py.
+ (CommonMethods.ExpectHeader): Move to conf/expect_header.py.
+ (CommonMethods.RejectHeader): Move to conf/reject_header.py.
+ (CommonMethods.Response): Move to conf/response.py.
+ (CommonMethods.SendHeader): Move to conf/send_header.py.
+ (CommonMethods.ServerFiles): Move to conf/server_files.py.
+ (CommonMethods.LocalFiles): Move to conf/local_files.py.
+ (CommonMethods.ServerConf): Move to conf/server_conf.py.
+ (CommonMethods.WgetCommands): Move to conf/wget_commands.py.
+ (CommonMethods.Urls): Move to conf/urls.py.
+ (CommonMethods.ExpectedRetcode): Move to conf/expected_retcode.py.
+ (CommonMethods.ExpectedFiles): Move to conf/expected_files.py.
+ (CommonMethods.FilesCrawled): Move to conf/files_crawled.py.
+ (CommonMethods.__check_downloaded_files): Rename to
+ _check_downloaded_files, so that the method is callable from outside the
+ class.
+ (CommomMethods.get_server_rules): Modify so that it utilizes the conf
+ package.
+ (HTTPTest): Add a method hook_call(configs, name) to reduce duplications
+ in pre_hook_call, call_test and post_hook_call utilizing the conf package.
+ * conf/hook_sample.py: (new file) sample for hooks
+ * conf/rule_sample.py: (new file) sample for rules
+ * REAMDE: Update sections about customizing rules and hooks.
+
+2014-03-13 Zihang Chen <chsc4698@gmail.com>
+
+ * exc: (new package) package for miscellaneous exceptions
+ * WgetTest.py: Move TestFailed to exc/test_failed.py.
+
+2014-03-13 Zihang Chen <chsc4698@gmail.com>
+
+ * Test-Proto.py: Fix a typo (line 71: server to servers).
+
+2014-03-13 Zihang Chen <chsc4698@gmail.com>
+
+ * WgetTest.py: Move WgetFile to package misc.
+ * README: Modify documentation respect to WgetFile.
+ * Test-*.py: Optimize imports about WgetFile.
+
+2014-03-13 Zihang Chen <chsc4698@gmail.com>
+
+ * misc: (new package) package for miscellaneous modules
+ * ColourTerm.py: Move to package misc and rename to colour_terminal.py,
+ add print_color functions to reduce the use of string literals like
+ "BLUE", "RED" etc.
+ * WgetTest.py:
+ (CommonMethods.Server_setup): Change invocation to printer to print_blue.
+ (CommonMethods.FilesCrawled): Change invocation to printer to print_red.
+ (HTTPTest.__init__): Change invocations to printer to print_red and
+ print_green respectively.
+
+2014-01-02 Darshit Shah <darnir@gmail.com>
+ * Makefile.am: Add new Test--https.py to list of tests and EXTRA_DIST.
+ Also replace all tabs with spaces in file for conformity.
+ * Test--https.py: New test to check if Wget works correctly with HTTPS
+ servers
+ * HTTPServer.py: Import new modules for use in HTTPS Servers
+ (HTTPSServer): New class that generates a SSL-wrapped socket for use in a
+ HTTPS Server.
+ (HTTPSd): HTTPS daemon class. Analogous to the HTTPd class
+ * WgetTest.py: Define global variables HTTP and HTTPS to reflect Server
+ types
+ (CommonMethods.exec_wget): Add the protocol information to the URL before
+ passing it to wget
+ (HTTPTest.__init__): Edit syntax. The servers variable now accepts a list of
+ servers defined by their type. E.g. HTTP, HTTPS.
+ (HTTPTest.Server_setup): Reflect change in type of variable servers.
+ However, we maintin the value of self.servers to allow most of the code to
+ remain unchanged.
+ (HTTPTest.init_HTTPS_Server): Initialize a HTTPS Server
+ * Test-Parallel-Proto.py: Edit to reflect slight change in Test Fiel Syntax.
+ * Test-Proto.py: Same
+
+2014-01-02 Darshit Shah <darnir@gmail.com>
+
+ * WgetTest.py (CommonMentods.exec_wget): Wait for n seconds before calling
+ the Wget executable.
+
+2013-12-27 Darshit Shah <darnir@gmail.com>
+
+ * WgetTest.py: Add modeline
+ (CommonMethods.ServerConf): New pre-test hook that sets
+ BaseHTTPRequestHandler class variables in all available servers
+ * HTTPServer.py (HTTPd.ServerConf): Call the respective method in the Server
+ to set the class variables
+ (StoppableHTTPServer.server_sett): Set the handler class variables
+
+2013-12-26 Darshit Shah <darnir@gmail.com>
+
+ * WgetTest.py (HTTPTest.call_test): Correct the call to stop_HTTP_Server.
+
+2013-12-25 Darshit Shah <darnir@gmail.com>
+
+ * WgetTest.py (CommonMehtods.exec_wget): Catch and handle exception if the
+ Wget executable is not found at src/wget
+ (HTTPTest.call_test): In case of error during execution, remove all existing
+ servers before quitting
+
+2013-12-15 Darshit Shah <darnir@gmail.com>
+
+ * WgetTest.py (HTTPTest.HTTP_setup): Rename to Server_setup so it can be
+ easily reused for other non-HTTP servers.
+ (HTTPTest.__init__): Call Server_setup instead of HTTP_setup
+ (HTTPTest.Server_setup): Split into three more functions, that handle
+ pre-hooks, test execution and post-hooks respectively.
+ (HTTPTest.pre_hook_call): Set up and execute the pre-test hooks. Code split
+ from HTTPTest.Server_setup
+ (HTTPTest.call_test): Execute wget and log exit code. Code split from
+ HTTPTest.Server_setup
+ (HTTPTest.post_hook_call): Set up and execute post-test hooks. Code split
+ from HTTPTest.Server_setup
+
+2013-10-14 Giuseppe Scrivano <gscrivan@redhat.com>
+
+ * Makefile.am (XFAIL_TESTS): Remove Test--spider-r.py.
+
+2013-10-06 Giuseppe Scrivano <gscrivan@redhat.com>
+
+ * Makefile.am (EXTRA_DIST): Distribute test files.
+
+2013-09-16 Darshit Shah <darnir@gmail.com>
+
+ * README: Update documentation
+
+2013-09-14 Darshit Shah <darnir@gmail.com>
+
+ * HTTPServer.py (StoppableHTTPServer): Define object variable
+ request_headers which stores a list of requests received by the server
+ (StoppableHTTPServer.get_req_headers): Return the list of Request
+ Headers stored by the server
+ (_Handler.do_HEAD): Send the Request MEthod string for identification
+ (_Handler.do_GET): Same
+ (_Handler.__log_request): Log the request in Request_Headers list
+ (_Handler.send_head): Make a call to __log_request
+ * Test--spider-r.py: Add new list, Request_List, which contains all
+ the requests that Wget is expected to send. This will allow for
+ fine-grained tests on recursive downloading.
+ * WgetTest.py (CommonMethods.FilesCrawled): New Post-Test Hook, that
+ ensures that all the expected Files on the server were accessed as
+ expected.
+ (HTTPTest.stop_HTTP_server): On stopping server, asks it to respond
+ with list of all requests it received.
+
+2013-09-13 Darshit Shah <darnir@gmail.com>
+
+ * Test--spider-r.py: Test retrieval in recursive spider mode.
+ * Makefile.am: add new file
+
+2013-09-13 Darshit Shah <darnir@gmail.com>
+
+ * HTTPServer.py (_Handler.do_HEAD): If requested path is /, respond
+ with /index.html
+ (_Handler.do_HEAD): Smartly guess value of Content-Type Header from
+ file extension
+ (_Handler.guess_type): Use a preset list of extensions and
+ Content-Type strings. If the extension matches one in the list, use
+ that string, else default to "text/plain"
+
+2013-09-13 Darshit Shah <darnir@gmail.com>
+
+ * WgetTest.py (CommonMethods._replace_substring): New method that will
+ replace a substring delimited by {{ }} characters by the value of
+ self.<substring> variable
+ (CommonMethods.WgetCommands): Use the _replace_substring () call to
+ replace the substrings in the the command line.
+ (CommonMethods.ServerFiles): Run the _replace_substring () method on
+ the File contents too.
+
+2013-09-11 Darshit Shah <darnir@gmail.com>
+
+ * WgetTest.py (CommonMethods.exec_wget): Expect domain_list instead of
+ domain.
+ (CommonMethods.get_cmd_line): Same. Generate command line by
+ prepending to each file it's respective domain string
+ (CommonMethods.ServerFiles): Generate file_list and server_rules for
+ each Server and set the config details
+ (HTTPTest): New named parameter, servers which signifies number of
+ servers to spawn
+ (HTTPTest.HTTP_setup): This method now takes servers as a new
+ parameter. Instead of storing server and domain, we now store
+ server_list and domain_list. Each server must be initialized through a
+ loop.
+ (HTTPTest.stop_HTTP_server): Stop all servers in a loop.
+ * Test-Parallel-Proto.py: Prototype test file for multiple servers.
+
+2013-09-10 Darshit Shah <darnir@gmail.com>
+
+ * WgetTest.py (HTTPTest.stop_HTTP_server): With the threaded servers,
+ we can simply use the socketserver.shutdown() method to close the
+ server instead of sending a QUIT command
+ * HTTPServer.py (StoppabelHTTPServer.serve_forever): Delete method. No
+ need to override this method anymore.
+ (WgetHTTPRequestHandler.do_QUIT): No longer required
+ (HTTPd): Rename self.server to self.server_inst to reduce ambiguity
+ when referenced from WgetTest
+
+2013-09-08 Darshit Shah <darnir@gmail.com>
+
+ * README (File Structure): Add explanation about various variables
+ used consistently across all tests.
+
+2013-09-07 Darshit Shah <darnir@gmail.com>
+
+ * HTTPServer.py: Remove bunch of old code artefacts
+ * WgetTest.py: Same
+
+2013-09-07 Darshit Shah <darnir@gmail.com>
+
+ * HTTPServer.py (StoppableHTTPServer.server_conf): Change global
+ variable fileSys to an object variable. This is good programming
+ practice and required for parallel-wget support.
+ (StoppableHTTPServer.server_forever): Edit overridden method to remove
+ the global queue variable. No longer required under the new working
+ (WgetHTTPRequestHandler.do_QUIT): Don't push fileSys through the queue
+ (_Handler): Rename class __Handler to _Handler to match Python's
+ encapsulation rules
+ (_Handler.do_POST): fileSys is now an object variable of the server
+ (_Handler.do_PUT): Same
+ (_Handler.send_put): Same
+ (_Handler.send_head): Same
+ (HTTPd): New class that wraps around the server for Threading
+ (create_server): Make new object of HTTPd.
+ (spawn_server): Start the thread created through create_server
+ (ret_fileSys): Removed method. No longer required.
+ * WgetTest.py (HTTPTest.__init__): Don't explicitly set
+ self.act_retcode. Instead toggle tests_passed boolean to set the
+ correct return code.
+ (HTTPTest.HTTP_setup): We no longer call HTTPServer.spawn_server to
+ start a new instance of the server.
+ (HTTPTest.init_HTTP_server): We no longer call the old
+ create_server(), spawn_server() methods. Instead use the new HTTPd
+ class interface to create new instances of the server
+ (HTTPTest.stop_HTTP_server): Don't ask server to return fileSys.
+
+2013-09-07 Darshit Shah <darnir@gmail.com>
+
+ * Test-Post.py: Test basic functionality for sending HTTP POST
+ requests using the --method command
+ * Makefile.am: Add new test
+
+2013-09-06 Darshit Shah <darnir@gmail.com>
+
+ * WgetTest.py (CommonMethods.__check_downloaded_files): Print a
+ unified diff in case there is a mismatch in the file contents
+
+2013-09-06 Darshit Shah <darnir@gmail.com>
+
+ * HTTPServer.py (WgetHTTPRequestHandler.test_cookies): Comment out the
+ old test_cookies code. This is no longer used and was causing problems
+ with expected cookies. The code will soon be removed anyways
+ * Test-cookie.py: Add new test for basic cookie functionality
+ * Test-cookie-401.py: Ensure cookies are saved during a 401 response
+ * Test-cookie-expires.py: Ensure that the Expires field is correctly
+ handled
+ * Test-cookies-domain-mismatch.py: Ensure that mismatched domains are
+ handled by Wget
+ * Makefile.am: Add the new tests
+
+2013-09-06 Darshit Shah <darnir@gmail.com>
+
+ * README: New section on pending work. Will keep updating this to keep
+ track of work that remains to be done on this implementation
+
+2013-09-05 Darshit Shah <darnir@gmail.com>
+
+ * Test-auth-with-content-disposition.py: Add test that ensures Content
+ Disposition works alongwith authentication
+ * Makefile.am: Add new test
+
+2013-09-04 Darshit Shah <darnir@gmail.com>
+
+ * Test-c-full.py: Test Continue options
+ * Makefile.am: Add Test-c-full.py and Test-O
+
+2013-09-02 Darshit Shah <darnir@gmail.com>
+
+ * Makefile.am: Add new Test
+ * Test-Head.py: New Test to ensure HEAD requests are handled correctly
+
+2013-08-31 Darshit Shah <darnir@gmail.com>
+
+ * README: Explain that TEST_NAME needs to be unique
+ * Test-auth-no-challenge.py: Edit non-unique TEST_NAME
+
+2013-08-31 Darshit Shah <darnir@gmail.com>
+
+ * HTTPTest.py (ServerError): Define new Exception for handling
+ internal control flow.
+ (StoppableHTTPServer.SendHeader): Simply pass. Do nothing. Adding
+ functionality here seems to crash for no apparent reason.
+ (stoppableHTTPServer.send_cust_headers): Minor optimization. No need
+ for extra variable.
+ (__Handler.Response): Handle explicit Response Code Rules
+ (__Handler.Authentication): Handle Authentication rules
+ (__Handler.handle_auth): Actual worker method for authentication
+ (__Handler.ExpectHeader): Ensure Expected Headers are received
+ (__Handler.RejectHeader): Ensure Blacklisted Headers are NOT received
+ (__Handler.send_HEAD): Dynamically call server rule functions based on
+ the self.rules list. This feature will later be added to POST/PUT, etc
+
+2013-08-31 Darshit Shah <darnir@gmail.com>
+
+ * WgetTest.py: Remove import module defaultdict.
+ (CommonMethods.get_server_rules): server_rules should be a dict, not a
+ defaultdict (list).
+ * HTTPServer.py (WgetHTTPRequestHandler.get_rule_list): If rule does
+ not exist, return None. Not an emppty list.
+ (WgetHTTPRequestHandler.test_cookies): Rule variable is not a list
+ (__Handler.send_cust_headers): Same
+ (__Handler.custom_response): Same
+ (__Handler.is_authorized): Same
+ (__Handler.expect_headers): Same
+ (__Handler.reject_headers): Same
+
+2013-08-31 Darshit Shah <darnir@gmail.com>
+
+ * README: (newfile) Simple help / instructions about using the Test
+ Environment.
+ * Makefile.am: (newfile) Makefile for the Test Environment. Uses the
+ Automake Parallel Test Harness
+ * WgetTest.py: (newfile) Base module that executes the Test.
+ * HTTPServer.py: (newfile) Contains the custom HTTP Server for the
+ Test Environment. Creates an instance of http.server in Python3.
+ * FTPServer.py: (newfile) Overrides methods from pyftpdlib for use in
+ the Test Environment. ** Work under progress **.
+ * ColourTerm.py: (newfile) A custom module to output coloured text to
+ the terminal. Known to work on POSIX shells.
+ * Test-Proto.py: (newfile) A prototype Test File. This should be
+ copied when writing a new Test Case.
+ * Test-Content-disposition-2.py: Test Content Disposition clobbering
+ * Test-Content-disposition.py: Test Content Disposition Headers
+ * Test-O.py: Test Output filename command
+ * Test-auth-basic-fail.py: Test returncode on auth failure
+ * Test-auth-basic.py: Test Basic Auth negotiation
+ * Test-auth-both.py: Test handling of Multiple auth providers. This
+ test currently fails.
+ * Test-auth-digest.py: Test Digest Auth Negotiation
+ * Test-auth-no-challenge-url.py: Ensure --auth-no-challenge is handled
+ when auth details are in-URL.
+ * Test-auth-no-challenge.py: Ensure --auth-no-challenge is honoured
+ * Test-auth-retcode.py: Ensure correct return code after 403 Forbidden
+ response.
diff --git a/testenv/Test--https.py b/testenv/Test--https.py
index 55f417be..17f49d3d 100755
--- a/testenv/Test--https.py
+++ b/testenv/Test--https.py
@@ -17,7 +17,7 @@ A_File = WgetFile ("File1", File1)
B_File = WgetFile ("File2", File2)
C_File = WgetFile ("File3", File3)
-WGET_OPTIONS = "-d --no-check-certificate"
+WGET_OPTIONS = "--no-check-certificate"
WGET_URLS = [["File1", "File2"]]
Files = [[A_File, B_File]]
diff --git a/testenv/Test--spider-r.py b/testenv/Test--spider-r.py
index df023d3a..5eb01e46 100755
--- a/testenv/Test--spider-r.py
+++ b/testenv/Test--spider-r.py
@@ -74,7 +74,7 @@ Request_List = [
]
]
-WGET_OPTIONS = "-d --spider -r"
+WGET_OPTIONS = "--spider -r"
WGET_URLS = [[""]]
Files = [[index_html, secondpage_html, thirdpage_html, dummy_txt]]
diff --git a/testenv/Test-Content-disposition-2.py b/testenv/Test-Content-disposition-2.py
index 3bf49405..5b9cf91a 100755
--- a/testenv/Test-Content-disposition-2.py
+++ b/testenv/Test-Content-disposition-2.py
@@ -20,7 +20,7 @@ File2_rules = {
A_File = WgetFile ("HTTP.Teapot", File1)
B_File = WgetFile ("File2", File2, rules=File2_rules)
-WGET_OPTIONS = "-d --content-disposition"
+WGET_OPTIONS = "--content-disposition"
WGET_URLS = [["File2"]]
Files = [[B_File]]
diff --git a/testenv/Test-Content-disposition.py b/testenv/Test-Content-disposition.py
index ce245994..027a9e8c 100755
--- a/testenv/Test-Content-disposition.py
+++ b/testenv/Test-Content-disposition.py
@@ -25,7 +25,7 @@ File1_rules = {
}
A_File = WgetFile ("LOTR", File1, rules=File1_rules)
-WGET_OPTIONS = "-d --content-disposition"
+WGET_OPTIONS = "--content-disposition"
WGET_URLS = [["LOTR"]]
Files = [[A_File]]
diff --git a/testenv/Test-Head.py b/testenv/Test-Head.py
index e3562529..77e5c575 100755
--- a/testenv/Test-Head.py
+++ b/testenv/Test-Head.py
@@ -13,7 +13,7 @@ File1 = "You shall not pass!"
A_File = WgetFile ("File1", File1)
-WGET_OPTIONS = "-d --method=HEAD"
+WGET_OPTIONS = "--method=HEAD"
WGET_URLS = [["File1"]]
Files = [[A_File]]
diff --git a/testenv/Test-O.py b/testenv/Test-O.py
index 784a229d..90cb146a 100755
--- a/testenv/Test-O.py
+++ b/testenv/Test-O.py
@@ -13,7 +13,7 @@ File1 = "Test Contents."
A_File = WgetFile ("File1", File1)
-WGET_OPTIONS = "-d -O NewFile.txt"
+WGET_OPTIONS = "-O NewFile.txt"
WGET_URLS = [["File1"]]
Files = [[A_File]]
diff --git a/testenv/Test-Parallel-Proto.py b/testenv/Test-Parallel-Proto.py
index b5e42bbe..beb768c5 100755
--- a/testenv/Test-Parallel-Proto.py
+++ b/testenv/Test-Parallel-Proto.py
@@ -18,7 +18,7 @@ A_File = WgetFile ("File1", File1)
B_File = WgetFile ("File2", File2)
C_File = WgetFile ("File3", File3)
-WGET_OPTIONS = "-d"
+WGET_OPTIONS = ""
WGET_URLS = [["File1"], ["File2"]]
Files = [[A_File], [B_File]]
diff --git a/testenv/Test-Post.py b/testenv/Test-Post.py
index 8983454b..223641af 100755
--- a/testenv/Test-Post.py
+++ b/testenv/Test-Post.py
@@ -17,7 +17,7 @@ TestMessage"""
A_File = WgetFile ("File1", File1)
-WGET_OPTIONS = "-d --method=post --body-data=TestMessage"
+WGET_OPTIONS = "--method=post --body-data=TestMessage"
WGET_URLS = [["File1"]]
Files = [[A_File]]
diff --git a/testenv/Test-Proto.py b/testenv/Test-Proto.py
index d26b2bb3..cc71fb9d 100755
--- a/testenv/Test-Proto.py
+++ b/testenv/Test-Proto.py
@@ -39,7 +39,7 @@ A_File = WgetFile ("File1", File1, rules=File1_rules)
B_File = WgetFile ("File2", File2, rules=File2_rules)
C_File = WgetFile ("File3", File3)
-WGET_OPTIONS = "-d --content-disposition --user=Sauron --password=TheEye"
+WGET_OPTIONS = "--content-disposition --user=Sauron --password=TheEye"
WGET_URLS = [["File1", "File2"]]
Servers = [HTTP]
diff --git a/testenv/Test-auth-basic-fail.py b/testenv/Test-auth-basic-fail.py
index 263f7566..0badf693 100755
--- a/testenv/Test-auth-basic-fail.py
+++ b/testenv/Test-auth-basic-fail.py
@@ -20,7 +20,7 @@ File1_rules = {
}
A_File = WgetFile ("File1", File1, rules=File1_rules)
-WGET_OPTIONS = "-d --user=Sauron --password=Eye"
+WGET_OPTIONS = "--user=Sauron --password=Eye"
WGET_URLS = [["File1"]]
Files = [[A_File]]
diff --git a/testenv/Test-auth-basic.py b/testenv/Test-auth-basic.py
index 102bf8c6..49bb27b2 100755
--- a/testenv/Test-auth-basic.py
+++ b/testenv/Test-auth-basic.py
@@ -28,7 +28,7 @@ File2_rules = {
A_File = WgetFile ("File1", File1, rules=File1_rules)
B_File = WgetFile ("File2", File2, rules=File2_rules)
-WGET_OPTIONS = "-d --user=Sauron --password=TheEye"
+WGET_OPTIONS = "--user=Sauron --password=TheEye"
WGET_URLS = [["File1", "File2"]]
Files = [[A_File, B_File]]
diff --git a/testenv/Test-auth-both.py b/testenv/Test-auth-both.py
index 2da2840f..91d72b84 100755
--- a/testenv/Test-auth-both.py
+++ b/testenv/Test-auth-both.py
@@ -46,7 +46,7 @@ A_File = WgetFile ("File1", File1, rules=File1_rules)
B_File = WgetFile ("File2", File2, rules=File2_rules)
C_File = WgetFile ("File3", File3, rules=File3_rules)
-WGET_OPTIONS = "-d --user=Sauron --password=TheEye"
+WGET_OPTIONS = "--user=Sauron --password=TheEye"
WGET_URLS = [["File1", "File2", "File3"]]
Files = [[A_File, B_File, C_File]]
diff --git a/testenv/Test-auth-digest.py b/testenv/Test-auth-digest.py
index f6d28c7c..8a73c0d5 100755
--- a/testenv/Test-auth-digest.py
+++ b/testenv/Test-auth-digest.py
@@ -19,7 +19,7 @@ File1_rules = {
}
A_File = WgetFile ("File1", File1, rules=File1_rules)
-WGET_OPTIONS = "-d --user=Pacman --password=Omnomnom"
+WGET_OPTIONS = "--user=Pacman --password=Omnomnom"
WGET_URLS = [["File1"]]
Files = [[A_File]]
diff --git a/testenv/Test-auth-no-challenge-url.py b/testenv/Test-auth-no-challenge-url.py
index c39ebaa9..9e062600 100755
--- a/testenv/Test-auth-no-challenge-url.py
+++ b/testenv/Test-auth-no-challenge-url.py
@@ -23,7 +23,7 @@ File1_rules = {
}
A_File = WgetFile ("File1", File1, rules=File1_rules)
-WGET_OPTIONS = "-d --auth-no-challenge http://Pacman:Omnomnom@localhost:{{port}}/File1"
+WGET_OPTIONS = "--auth-no-challenge http://Pacman:Omnomnom@localhost:{{port}}/File1"
WGET_URLS = [[]]
Files = [[A_File]]
diff --git a/testenv/Test-auth-no-challenge.py b/testenv/Test-auth-no-challenge.py
index f02c0307..d155d725 100755
--- a/testenv/Test-auth-no-challenge.py
+++ b/testenv/Test-auth-no-challenge.py
@@ -23,7 +23,7 @@ File1_rules = {
}
A_File = WgetFile ("File1", File1, rules=File1_rules)
-WGET_OPTIONS = "-d --auth-no-challenge --user=Pacman --password=Omnomnom"
+WGET_OPTIONS = "--auth-no-challenge --user=Pacman --password=Omnomnom"
WGET_URLS = [["File1"]]
Files = [[A_File]]
diff --git a/testenv/Test-auth-retcode.py b/testenv/Test-auth-retcode.py
index 8719bd01..13eb583b 100755
--- a/testenv/Test-auth-retcode.py
+++ b/testenv/Test-auth-retcode.py
@@ -19,7 +19,7 @@ File1_rules = {
A_File = WgetFile ("File1", File1, rules=File1_rules)
-WGET_OPTIONS = "-d"
+WGET_OPTIONS = ""
WGET_URLS = [["File1"]]
Files = [[A_File]]
diff --git a/testenv/Test-auth-with-content-disposition.py b/testenv/Test-auth-with-content-disposition.py
index f74a9592..df28c91e 100755
--- a/testenv/Test-auth-with-content-disposition.py
+++ b/testenv/Test-auth-with-content-disposition.py
@@ -23,7 +23,7 @@ File1_rules = {
}
A_File = WgetFile ("File1", File1, rules=File1_rules)
-WGET_OPTIONS = "-d --user=Pacman --password=Omnomnom --content-disposition"
+WGET_OPTIONS = "--user=Pacman --password=Omnomnom --content-disposition"
WGET_URLS = [["File1"]]
Files = [[A_File]]
diff --git a/testenv/Test-c-full.py b/testenv/Test-c-full.py
index 3cdcd768..6dc3f5e2 100755
--- a/testenv/Test-c-full.py
+++ b/testenv/Test-c-full.py
@@ -20,7 +20,7 @@ D_File = WgetFile ("File2", File2)
E_File = WgetFile ("File3", File1)
-WGET_OPTIONS = "-d -c"
+WGET_OPTIONS = "-c"
WGET_URLS = [["File1", "File2", "File3"]]
Files = [[A_File, C_File, E_File]]
diff --git a/testenv/Test-cookie-401.py b/testenv/Test-cookie-401.py
index 9488c34e..4ffb1ff9 100755
--- a/testenv/Test-cookie-401.py
+++ b/testenv/Test-cookie-401.py
@@ -28,7 +28,7 @@ File2_rules = {
A_File = WgetFile ("File1", File1, rules=File1_rules)
B_File = WgetFile ("File2", File2, rules=File2_rules)
-WGET_OPTIONS = "-d"
+WGET_OPTIONS = ""
WGET_URLS = [["File1", "File2"]]
Files = [[A_File, B_File]]
diff --git a/testenv/Test-cookie-domain-mismatch.py b/testenv/Test-cookie-domain-mismatch.py
index 92487f43..45c5f1de 100755
--- a/testenv/Test-cookie-domain-mismatch.py
+++ b/testenv/Test-cookie-domain-mismatch.py
@@ -26,7 +26,7 @@ File2_rules = {
A_File = WgetFile ("File1", File1, rules=File1_rules)
B_File = WgetFile ("File2", File2, rules=File2_rules)
-WGET_OPTIONS = "-d"
+WGET_OPTIONS = ""
WGET_URLS = [["File1", "File2"]]
Files = [[A_File, B_File]]
diff --git a/testenv/Test-cookie-expires.py b/testenv/Test-cookie-expires.py
index 48a93b99..7ad919d5 100755
--- a/testenv/Test-cookie-expires.py
+++ b/testenv/Test-cookie-expires.py
@@ -49,7 +49,7 @@ B_File = WgetFile ("File2", File2, rules=File2_rules)
C_File = WgetFile ("File3", File3, rules=File3_rules)
D_File = WgetFile ("File4", File4, rules=File4_rules)
-WGET_OPTIONS = "-d"
+WGET_OPTIONS = ""
WGET_URLS = [["File1", "File2", "File3", "File4"]]
Files = [[A_File, B_File, C_File, D_File]]
diff --git a/testenv/Test-cookie.py b/testenv/Test-cookie.py
index b70316d8..13935181 100755
--- a/testenv/Test-cookie.py
+++ b/testenv/Test-cookie.py
@@ -26,7 +26,7 @@ File2_rules = {
A_File = WgetFile ("File1", File1, rules=File1_rules)
B_File = WgetFile ("File2", File2, rules=File2_rules)
-WGET_OPTIONS = "-d"
+WGET_OPTIONS = ""
WGET_URLS = [["File1", "File2"]]
Files = [[A_File, B_File]]
diff --git a/testenv/conf/__init__.py b/testenv/conf/__init__.py
index 156e9b60..603bd62b 100644
--- a/testenv/conf/__init__.py
+++ b/testenv/conf/__init__.py
@@ -3,6 +3,7 @@ import os
# this file implements the mechanism of conf class auto-registration,
# don't modify this file if you have no idea what you're doing
+
def gen_hook():
hook_table = {}
diff --git a/testenv/conf/authentication.py b/testenv/conf/authentication.py
index 58cbaff0..c87994ad 100644
--- a/testenv/conf/authentication.py
+++ b/testenv/conf/authentication.py
@@ -1,5 +1,18 @@
from conf import rule
+""" Rule: Authentication
+This file defines an authentication rule which when applied to any file will
+cause the server to prompt the client for the required authentication details
+before serving it.
+auth_type must be either of: Basic, Digest, Both or Both-inline
+When auth_type is Basic or Digest, the server asks for the respective
+authentication in its response. When auth_type is Both, the server sends two
+Authenticate headers, one requesting Basic and the other requesting Digest
+authentication. If auth_type is Both-inline, the server sends only one
+Authenticate header, but lists both Basic and Digest as supported mechanisms in
+that.
+"""
+
@rule()
class Authentication:
diff --git a/testenv/conf/expect_header.py b/testenv/conf/expect_header.py
index 87b0e24d..055099fa 100644
--- a/testenv/conf/expect_header.py
+++ b/testenv/conf/expect_header.py
@@ -1,5 +1,10 @@
from conf import rule
+""" Rule: ExpectHeader
+This rule defines a dictionary of headers and their value which the server
+should expect in each request for the file to which the rule was applied.
+"""
+
@rule()
class ExpectHeader:
diff --git a/testenv/conf/expected_files.py b/testenv/conf/expected_files.py
index a8b2ee19..2c8d632c 100644
--- a/testenv/conf/expected_files.py
+++ b/testenv/conf/expected_files.py
@@ -4,6 +4,15 @@ import sys
from conf import hook
from exc.test_failed import TestFailed
+""" Post-Test Hook: ExpectedFiles
+This is a Post-Test hook that checks the test directory for the files it
+contains. A dictionary object is passed to it, which contains a mapping of
+filenames and contents of all the files that the directory is expected to
+contain.
+Raises a TestFailed exception if the expected files are not found or if extra
+files are found, else returns gracefully.
+"""
+
@hook()
class ExpectedFiles:
@@ -34,7 +43,7 @@ class ExpectedFiles:
fromfile='Actual',
tofile='Expected'):
print(line, file=sys.stderr)
- raise TestFailed('Contents of %s do not match.' % file.name)
+ raise TestFailed('Contents of %s do not match' % file.name)
else:
raise TestFailed('Expected file %s not found.' % file.name)
if local_fs:
diff --git a/testenv/conf/expected_ret_code.py b/testenv/conf/expected_ret_code.py
index febef32f..87cba13b 100644
--- a/testenv/conf/expected_ret_code.py
+++ b/testenv/conf/expected_ret_code.py
@@ -1,6 +1,14 @@
from exc.test_failed import TestFailed
from conf import hook
+""" Post-Test Hook: ExpectedRetCode
+This is a post-test hook which checks if the exit code of the Wget instance
+under test is the same as that expected. As a result, this is a very important
+post test hook which is checked in all the tests.
+Returns a TestFailed exception if the return code does not match the expected
+value. Else returns gracefully.
+"""
+
@hook(alias='ExpectedRetcode')
class ExpectedRetCode:
diff --git a/testenv/conf/files_crawled.py b/testenv/conf/files_crawled.py
index 3f52008d..334e5964 100644
--- a/testenv/conf/files_crawled.py
+++ b/testenv/conf/files_crawled.py
@@ -2,6 +2,15 @@ from misc.colour_terminal import print_red
from conf import hook
from exc.test_failed import TestFailed
+""" Post-Test Hook: FilesCrawled
+This is a post test hook that is invoked in tests that check wget's behaviour
+in recursive mode. It expects an ordered list of the request lines that Wget
+must send to the server. If the requests received by the server do not match
+the provided list, IN THE GIVEN ORDER, then it raises a TestFailed exception.
+Such a test can be used to check the implementation of the recursion algorithm
+in Wget too.
+"""
+
@hook()
class FilesCrawled:
diff --git a/testenv/conf/hook_sample.py b/testenv/conf/hook_sample.py
index f48942f0..6230a706 100644
--- a/testenv/conf/hook_sample.py
+++ b/testenv/conf/hook_sample.py
@@ -1,7 +1,12 @@
from exc.test_failed import TestFailed
from conf import hook
-# this file is a hook example
+""" Hook: SampleHook
+This a sample file for how a new hook should be defined.
+Any errors should always be reported by raising a TestFailed exception instead
+of returning a true or false value.
+"""
+
@hook(alias='SampleHookAlias')
class SampleHook:
@@ -12,4 +17,6 @@ class SampleHook:
def __call__(self, test_obj):
# implement hook here
# if you need the test case instance, refer to test_obj
+ if False:
+ raise TestFailed ("Reason")
pass
diff --git a/testenv/conf/local_files.py b/testenv/conf/local_files.py
index 1eb3e4e5..5f9c8fad 100644
--- a/testenv/conf/local_files.py
+++ b/testenv/conf/local_files.py
@@ -1,5 +1,11 @@
from conf import hook
+""" Pre-Test Hook: LocalFiles
+This is a pre-test hook used to generate the specific environment before a test
+is run. The LocalFiles hook creates the files which should exist on disk before
+invoking Wget.
+"""
+
@hook()
class LocalFiles:
diff --git a/testenv/conf/reject_header.py b/testenv/conf/reject_header.py
index 1f451456..53e237dd 100644
--- a/testenv/conf/reject_header.py
+++ b/testenv/conf/reject_header.py
@@ -1,5 +1,11 @@
from conf import rule
+""" Rule: RejectHeader
+This is a server side rule which expects a dictionary object of Headers and
+their values which should be blacklisted by the server for a particular file's
+requests.
+"""
+
@rule()
class RejectHeader:
diff --git a/testenv/conf/response.py b/testenv/conf/response.py
index 23d55de5..976a9ce9 100644
--- a/testenv/conf/response.py
+++ b/testenv/conf/response.py
@@ -1,5 +1,9 @@
from conf import rule
+""" Rule: Response
+When this rule is set against a certain file, the server will unconditionally
+respond to any request for the said file with the provided response code. """
+
@rule()
class Response:
diff --git a/testenv/conf/send_header.py b/testenv/conf/send_header.py
index 61dbc0ec..1ac54cc4 100644
--- a/testenv/conf/send_header.py
+++ b/testenv/conf/send_header.py
@@ -1,5 +1,10 @@
from conf import rule
+""" Rule: SendHeader
+Have the server send custom headers when responding to a request for the file
+this rule is applied to. The header_obj object is expected to be dictionary
+mapping headers to their contents. """
+
@rule()
class SendHeader:
diff --git a/testenv/conf/server_files.py b/testenv/conf/server_files.py
index bf6c1633..1e9d3466 100644
--- a/testenv/conf/server_files.py
+++ b/testenv/conf/server_files.py
@@ -1,5 +1,16 @@
from conf import hook
+""" Pre-Test Hook: ServerFiles
+This hook is used to define a set of files on the server's virtual filesystem.
+server_files is expected to be dictionary that maps filenames to their
+contents. In the future, this can be used to add additional metadat to the
+files using the WgetFile class too.
+
+This hook also does some additional processing on the contents of the file. Any
+text between {{and}} is replaced by the contents of a class variable of the
+same name. This is useful in creating files that contain an absolute link to
+another file on the same server. """
+
@hook()
class ServerFiles:
diff --git a/testenv/conf/urls.py b/testenv/conf/urls.py
index 60015867..f34c13eb 100644
--- a/testenv/conf/urls.py
+++ b/testenv/conf/urls.py
@@ -1,5 +1,9 @@
from conf import hook
+""" Pre-Test Hook: URLS
+This hook is used to define the paths of the files on the test server that wget
+will send a request for. """
+
@hook(alias='Urls')
class URLs:
diff --git a/testenv/conf/wget_commands.py b/testenv/conf/wget_commands.py
index a326bb56..2b7522ec 100644
--- a/testenv/conf/wget_commands.py
+++ b/testenv/conf/wget_commands.py
@@ -1,5 +1,10 @@
from conf import hook
+""" Pre-Test Hook: WgetCommands
+This hook is used to specify the test specific switches that must be passed to
+wget on invokation. Default switches are hard coded in the test suite itself.
+"""
+
@hook()
class WgetCommands:
diff --git a/testenv/exc/server_error.py b/testenv/exc/server_error.py
new file mode 100644
index 00000000..b8a37ce7
--- /dev/null
+++ b/testenv/exc/server_error.py
@@ -0,0 +1,7 @@
+
+class ServerError (Exception):
+ """ A custom exception which is raised by the test servers. Often used to
+ handle control flow. """
+
+ def __init__ (self, err_message):
+ self.err_message = err_message
diff --git a/testenv/misc/colour_terminal.py b/testenv/misc/colour_terminal.py
index 206ffa30..cfbae94e 100644
--- a/testenv/misc/colour_terminal.py
+++ b/testenv/misc/colour_terminal.py
@@ -2,24 +2,39 @@ from functools import partial
import platform
from os import getenv
+""" This module allows printing coloured output to the terminal when running a
+Wget Test under certain conditions.
+The output is coloured only on Linux systems. This is because coloured output
+in the terminal on Windows requires too much effort for what is simply a
+convenience. This might work on OSX terminals, but without a confirmation, it
+remains unsupported.
+
+Another important aspect is that the coloured output is printed only if the
+environment variable MAKE_CHECK is not set. This variable is set when running
+the test suite through, `make check`. In that case, the output is not only
+printed to the terminal but also copied to a log file where the ANSI escape
+codes on;y add clutter. """
+
+
T_COLORS = {
- 'PURPLE' : '\033[95m',
- 'BLUE' : '\033[94m',
- 'GREEN' : '\033[92m',
- 'YELLOW' : '\033[93m',
- 'RED' : '\033[91m',
- 'ENDC' : '\033[0m'
+ 'PURPLE' : '\033[95m',
+ 'BLUE' : '\033[94m',
+ 'GREEN' : '\033[92m',
+ 'YELLOW' : '\033[93m',
+ 'RED' : '\033[91m',
+ 'ENDC' : '\033[0m'
}
-def printer (color, string):
- if platform.system () == 'Linux':
- if getenv ("MAKE_CHECK", "False") == "True":
- print (string)
- else:
- print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC'))
- else:
- print (string)
+def printer (color, string):
+ if platform.system () == 'Linux':
+ if getenv ("MAKE_CHECK", "False") == "True":
+ print (string)
+ else:
+ print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC'))
+
+ else:
+ print (string)
print_blue = partial(printer, 'BLUE')
@@ -28,4 +43,4 @@ print_green = partial(printer, 'GREEN')
print_purple = partial(printer, 'PURPLE')
print_yellow = partial(printer, 'YELLOW')
-# vim: set ts=8 sw=3 tw=0 et :
+# vim: set ts=8 sw=3 tw=80 et :
diff --git a/testenv/server/http/http_server.py b/testenv/server/http/http_server.py
index e554a105..12e04348 100644
--- a/testenv/server/http/http_server.py
+++ b/testenv/server/http/http_server.py
@@ -1,4 +1,5 @@
from http.server import HTTPServer, BaseHTTPRequestHandler
+from exc.server_error import ServerError
from socketserver import BaseServer
from posixpath import basename, splitext
from base64 import b64encode
@@ -11,20 +12,12 @@ import ssl
import os
-class InvalidRangeHeader (Exception):
-
- """ Create an Exception for handling of invalid Range Headers. """
- # TODO: Eliminate this exception and use only ServerError
-
- def __init__ (self, err_message):
- self.err_message = err_message
-
-class ServerError (Exception):
- def __init__ (self, err_message):
- self.err_message = err_message
-
-
class StoppableHTTPServer (HTTPServer):
+ """ This class extends the HTTPServer class from default http.server library
+ in Python 3. The StoppableHTTPServer class is capable of starting an HTTP
+ server that serves a virtual set of files made by the WgetFile class and
+ has most of its properties configurable through the server_conf()
+ method. """
request_headers = list ()
@@ -35,45 +28,45 @@ class StoppableHTTPServer (HTTPServer):
self.server_configs = conf_dict
self.fileSys = filelist
- def server_sett (self, settings):
- for settings_key in settings:
- setattr (self.RequestHandlerClass, settings_key, settings[settings_key])
-
def get_req_headers (self):
return self.request_headers
+
class HTTPSServer (StoppableHTTPServer):
+ """ The HTTPSServer class extends the StoppableHTTPServer class with
+ additional support for secure connections through SSL. """
- def __init__ (self, address, handler):
- BaseServer.__init__ (self, address, handler)
- print (os.getcwd())
- CERTFILE = os.path.abspath (os.path.join ('..', 'certs', 'wget-cert.pem'))
- print (CERTFILE)
- fop = open (CERTFILE)
- print (fop.readline())
- self.socket = ssl.wrap_socket (
- sock = socket.socket (self.address_family, self.socket_type),
- ssl_version = ssl.PROTOCOL_TLSv1,
- certfile = CERTFILE,
- server_side = True
- )
- self.server_bind ()
- self.server_activate ()
+ def __init__ (self, address, handler):
+ BaseServer.__init__ (self, address, handler)
+ print (os.getcwd())
+ CERTFILE = os.path.abspath(os.path.join('..', 'certs', 'wget-cert.pem'))
+ print (CERTFILE)
+ fop = open (CERTFILE)
+ print (fop.readline())
+ self.socket = ssl.wrap_socket (
+ sock = socket.socket (self.address_family, self.socket_type),
+ ssl_version = ssl.PROTOCOL_TLSv1,
+ certfile = CERTFILE,
+ server_side = True
+ )
+ self.server_bind()
+ self.server_activate()
-class WgetHTTPRequestHandler (BaseHTTPRequestHandler):
- """ Define methods for handling Test Checks. """
+class _Handler (BaseHTTPRequestHandler):
+ """ This is a private class which tells the server *HOW* to handle each
+ request. For each HTTP Request Command that the server should be capable of
+ responding to, there must exist a do_REQUESTNAME() method which details the
+ steps in which such requests should be processed. The rest of the methods
+ in this class are auxilliary methods created to help in processing certain
+ requests. """
def get_rule_list (self, name):
r_list = self.rules.get (name) if name in self.rules else None
return r_list
-
-class _Handler (WgetHTTPRequestHandler):
-
- """ Define Handler Methods for different Requests. """
-
- InvalidRangeHeader = InvalidRangeHeader
+ # The defailt protocol version of the server we run is HTTP/1.1 not
+ # HTTP/1.0 which is the default with the http.server module.
protocol_version = 'HTTP/1.1'
""" Define functions for various HTTP Requests. """
@@ -82,6 +75,11 @@ class _Handler (WgetHTTPRequestHandler):
self.send_head ("HEAD")
def do_GET (self):
+ """ Process HTTP GET requests. This is the same as processing HEAD
+ requests and then actually transmitting the data to the client. If
+ send_head() does not specify any "start" offset, we send the complete
+ data, else transmit only partial data. """
+
content, start = self.send_head ("GET")
if content:
if start is None:
@@ -90,11 +88,26 @@ class _Handler (WgetHTTPRequestHandler):
self.wfile.write (content.encode ('utf-8')[start:])
def do_POST (self):
+ """ According to RFC 7231 sec 4.3.3, if the resource requested in a POST
+ request does not exist on the server, the first POST request should
+ create that resource. PUT requests are otherwise used to create a
+ resource. Hence, we call the handle for processing PUT requests if the
+ resource requested does not already exist.
+
+ Currently, when the server recieves a POST request for a resource, we
+ simply append the body data to the existing file and return the new
+ file to the client. If the file does not exist, a new file is created
+ using the contents of the request body. """
+
path = self.path[1:]
- self.rules = self.server.server_configs.get (path)
- if not self.custom_response ():
- return (None, None)
if path in self.server.fileSys:
+ self.rules = self.server.server_configs.get (path)
+ if not self.rules:
+ self.rules = dict ()
+
+ if not self.custom_response ():
+ return (None, None)
+
body_data = self.get_body_data ()
self.send_response (200)
self.send_header ("Content-type", "text/plain")
@@ -102,6 +115,7 @@ class _Handler (WgetHTTPRequestHandler):
total_length = len (content)
self.server.fileSys[path] = content
self.send_header ("Content-Length", total_length)
+ self.send_header ("Location", self.path)
self.finish_headers ()
try:
self.wfile.write (content.encode ('utf-8'))
@@ -115,7 +129,6 @@ class _Handler (WgetHTTPRequestHandler):
self.rules = self.server.server_configs.get (path)
if not self.custom_response ():
return (None, None)
- self.server.fileSys.pop (path, None)
self.send_put (path)
""" End of HTTP Request Method Handlers. """
@@ -126,12 +139,12 @@ class _Handler (WgetHTTPRequestHandler):
if header_line is None:
return None
if not header_line.startswith ("bytes="):
- raise InvalidRangeHeader ("Cannot parse header Range: %s" %
- (header_line))
+ raise ServerError ("Cannot parse header Range: %s" %
+ (header_line))
regex = re.match (r"^bytes=(\d*)\-$", header_line)
range_start = int (regex.group (1))
if range_start >= length:
- raise InvalidRangeHeader ("Range Overflow")
+ raise ServerError ("Range Overflow")
return range_start
def get_body_data (self):
@@ -141,23 +154,27 @@ class _Handler (WgetHTTPRequestHandler):
return body_data
def send_put (self, path):
+ if path in self.server.fileSys:
+ self.server.fileSys.pop (path, None)
+ self.send_response (204)
+ else:
+ self.rules = dict ()
+ self.send_response (201)
body_data = self.get_body_data ()
- self.send_response (201)
self.server.fileSys[path] = body_data
- self.send_header ("Content-type", "text/plain")
- self.send_header ("Content-Length", len (body_data))
+ self.send_header ("Location", self.path)
self.finish_headers ()
- try:
- self.wfile.write (body_data.encode ('utf-8'))
- except Exception:
- pass
+ """ This empty method is called automatically when all the rules are
+ processed for a given request. However, send_header() should only be called
+ AFTER a response has been sent. But, at the moment of processing the rules,
+ the appropriate response has not yet been identified. As a result, we defer
+ the processing of this rule till later. Each do_* request handler MUST call
+ finish_headers() instead of end_headers(). The finish_headers() method
+ takes care of sending the appropriate headers before completing the
+ response. """
def SendHeader (self, header_obj):
pass
-# headers_list = header_obj.headers
-# for header_line in headers_list:
-# print (header_line + " : " + headers_list[header_line])
-# self.send_header (header_line, headers_list[header_line])
def send_cust_headers (self):
header_obj = self.get_rule_list ('SendHeader')
@@ -195,11 +212,11 @@ class _Handler (WgetHTTPRequestHandler):
if auth_type == "Basic":
challenge_str = 'Basic realm="Wget-Test"'
elif auth_type == "Digest" or auth_type == "Both_inline":
- self.nonce = md5 (str (random ()).encode ('utf-8')).hexdigest ()
- self.opaque = md5 (str (random ()).encode ('utf-8')).hexdigest ()
- challenge_str = 'Digest realm="Test", nonce="%s", opaque="%s"' %(
- self.nonce,
- self.opaque)
+ self.nonce = md5 (str (random ()).encode ('utf-8')).hexdigest()
+ self.opaque = md5 (str (random ()).encode ('utf-8')).hexdigest()
+ challenge_str = 'Digest realm="Test", nonce="%s", opaque="%s"' % (
+ self.nonce,
+ self.opaque)
challenge_str += ', qop="auth"'
if auth_type == "Both_inline":
challenge_str = 'Basic realm="Wget-Test", ' + challenge_str
@@ -218,9 +235,9 @@ class _Handler (WgetHTTPRequestHandler):
n = len("Digest ")
auth_header = auth_header[n:].strip()
items = auth_header.split(", ")
- key_values = [i.split("=", 1) for i in items]
- key_values = [(k.strip(), v.strip().replace('"', '')) for k, v in key_values]
- return dict(key_values)
+ keyvals = [i.split("=", 1) for i in items]
+ keyvals = [(k.strip(), v.strip().replace('"', '')) for k, v in keyvals]
+ return dict(keyvals)
def KD (self, secret, data):
return self.H (secret + ":" + data)
@@ -237,10 +254,10 @@ class _Handler (WgetHTTPRequestHandler):
def check_response (self, params):
if "qop" in params:
data_str = params['nonce'] \
- + ":" + params['nc'] \
- + ":" + params['cnonce'] \
- + ":" + params['qop'] \
- + ":" + self.H (self.A2 (params))
+ + ":" + params['nc'] \
+ + ":" + params['cnonce'] \
+ + ":" + params['qop'] \
+ + ":" + self.H (self.A2 (params))
else:
data_str = params['nonce'] + ":" + self.H (self.A2 (params))
resp = self.KD (self.H (self.A1 ()), data_str)
@@ -256,11 +273,12 @@ class _Handler (WgetHTTPRequestHandler):
params = self.parse_auth_header (auth_header)
pass_auth = True
if self.user != params['username'] or \
- self.nonce != params['nonce'] or self.opaque != params['opaque']:
+ self.nonce != params['nonce'] or \
+ self.opaque != params['opaque']:
pass_auth = False
req_attribs = ['username', 'realm', 'nonce', 'uri', 'response']
for attrib in req_attribs:
- if not attrib in params:
+ if attrib not in params:
pass_auth = False
if not self.check_response (params):
pass_auth = False
@@ -326,19 +344,6 @@ class _Handler (WgetHTTPRequestHandler):
self.finish_headers ()
raise ServerError ("Header " + header_line + " not found")
- def expect_headers (self):
- """ This is modified code to handle a few changes. Should be removed ASAP """
- exp_headers_obj = self.get_rule_list ('ExpectHeader')
- if exp_headers_obj:
- exp_headers = exp_headers_obj.headers
- for header_line in exp_headers:
- header_re = self.headers.get (header_line)
- if header_re is None or header_re != exp_headers[header_line]:
- self.send_error (400, 'Expected Header not Found')
- self.end_headers ()
- return False
- return True
-
def RejectHeader (self, header_obj):
rej_headers = header_obj.headers
for header_line in rej_headers:
@@ -400,7 +405,7 @@ class _Handler (WgetHTTPRequestHandler):
try:
self.range_begin = self.parse_range_header (
self.headers.get ("Range"), content_length)
- except InvalidRangeHeader as ae:
+ except ServerError as ae:
# self.log_error("%s", ae.err_message)
if ae.err_message == "Range Overflow":
self.send_response (416)
@@ -431,9 +436,9 @@ class _Handler (WgetHTTPRequestHandler):
base_name = basename ("/" + path)
name, ext = splitext (base_name)
extension_map = {
- ".txt" : "text/plain",
- ".css" : "text/css",
- ".html" : "text/html"
+ ".txt" : "text/plain",
+ ".css" : "text/css",
+ ".html" : "text/html"
}
if ext in extension_map:
return extension_map[ext]
@@ -444,6 +449,7 @@ class _Handler (WgetHTTPRequestHandler):
class HTTPd (threading.Thread):
server_class = StoppableHTTPServer
handler = _Handler
+
def __init__ (self, addr=None):
threading.Thread.__init__ (self)
if addr is None:
@@ -452,16 +458,14 @@ class HTTPd (threading.Thread):
self.server_address = self.server_inst.socket.getsockname()[:2]
def run (self):
- self.server_inst.serve_forever ()
+ self.server_inst.serve_forever ()
def server_conf (self, file_list, server_rules):
self.server_inst.server_conf (file_list, server_rules)
- def server_sett (self, settings):
- self.server_inst.server_sett (settings)
class HTTPSd (HTTPd):
- server_class = HTTPSServer
+ server_class = HTTPSServer
# vim: set ts=4 sts=4 sw=4 tw=80 et :
diff --git a/testenv/test/base_test.py b/testenv/test/base_test.py
index 369370a3..8bf8ea5f 100644
--- a/testenv/test/base_test.py
+++ b/testenv/test/base_test.py
@@ -28,9 +28,9 @@ class BaseTest:
Attributes should not be defined outside __init__.
"""
self.name = name
- self.pre_configs = pre_hook or {} # if pre_hook == None, then
- # {} (an empty dict object) is
- # passed to self.pre_configs
+ self.pre_configs = pre_hook or {} # if pre_hook == None, then
+ # {} (an empty dict object) is
+ # passed to self.pre_configs
self.test_params = test_params or {}
self.post_configs = post_hook or {}
self.protocols = protocols
@@ -98,12 +98,13 @@ class BaseTest:
test_path = os.path.abspath(".")
wget_path = os.path.abspath(os.path.join(test_path,
"..", '..', 'src', "wget"))
+ wget_options = '--debug --no-config %s' % self.wget_options
if os.getenv("VALGRIND_TESTS"):
valgrind_test = "valgrind --error-exitcode=301 --leak-check=full"
else:
valgrind_test = ""
- cmd_line = '%s %s %s ' % (valgrind_test, wget_path, self.wget_options)
+ cmd_line = '%s %s %s ' % (valgrind_test, wget_path, wget_options)
for protocol, urls, domain in zip(self.protocols,
self.urls,
self.domains):
diff --git a/testenv/test/http_test.py b/testenv/test/http_test.py
index fe2254de..230eff8e 100644
--- a/testenv/test/http_test.py
+++ b/testenv/test/http_test.py
@@ -7,9 +7,10 @@ class HTTPTest(BaseTest):
""" Class for HTTP Tests. """
- # Temp Notes: It is expected that when pre-hook functions are executed, only an empty test-dir exists.
- # pre-hook functions are executed just prior to the call to Wget is made.
- # post-hook functions will be executed immediately after the call to Wget returns.
+ # Temp Notes: It is expected that when pre-hook functions are executed,
+ # only an empty test-dir exists. pre-hook functions are executed just prior
+ # to the call to Wget is made. post-hook functions will be executed
+ # immediately after the call to Wget returns.
def __init__(self,
name="Unnamed Test",