diff --git a/NEWS b/NEWS index 72f8728b..2153d9aa 100644 --- a/NEWS +++ b/NEWS @@ -9,6 +9,10 @@ Please send GNU Wget bug reports to . * Changes in Wget X.Y.Z +* When processing a Metalink file, --trust-server-names enables the + use of the destination file names specified in the Metalink file, + otherwise a safe destination file name is computed. + * When processing a Metalink file, enforce a safe destination path. Remove any drive letter prefix under w32, i.e. 'C:D:file'. Call libmetalink's metalink_check_safe_path() to prevent absolute, diff --git a/doc/metalink-standard.txt b/doc/metalink-standard.txt index d00c3843..18acaaae 100644 --- a/doc/metalink-standard.txt +++ b/doc/metalink-standard.txt @@ -29,18 +29,61 @@ paths or descend/escalate to a relative path unexpectedly. 2.1 Metalink/XML implemented tests ================================== -* testenv/Test-metalink-xml.py: Accept safe paths -* testenv/Test-metalink-xml-abspath.py: Reject absolute paths -* testenv/Test-metalink-xml-relpath.py: Reject relative paths -* testenv/Test-metalink-xml-homepath.py: Reject home paths +See testenv/Makefile.am (METALINK_TESTS). + +2.2 Metalink/HTTP implemented tests +=================================== + +See testenv/Makefile.am (METALINK_TESTS). 3. Download file name ********************* -Computing the file name to wrote from the followed urls only leads to -uncertainty. Reason why an unique name shall be used. Respectively, it -shall be the metalink:file "name" field for Metalink/XML and a derived -cli's url for Metalink/HTTP. +The download file name shall be decided by precise rules which prevent +any naming uncertainty and security issues. + +3.1 Naming rules +================ + +The final name of downloaded files is computed starting from a trusted +name, which is then combined with the "Directory Options". The result +is verified and eventually made safer following security rules. If the +final name isn't found safe enough, then the file isn't downloaded. + +Depending on the options used, a suffix could be appended to the final +name to not overwrite existing files. + +3.1.1 The trusted name +====================== + +The option --trust-server-names decides what is the trusted name. + +Any Metalink/XML element with an unsafe metalink:file "name" field is +ignored, see '1. Security features'. + +3.1.1.1 Without --trust-server-names +==================================== + +When --trust-server-names is off, the basename of the --input-metalink +file, if available, or of the mother URL is trusted. + +The files described by a Metalink/XML file will be named sequentially +applying a suffix to the trusted name. + +3.1.1.2 With --trust-server-names +================================= + +When --trust-server-names is on, the metalink:file "name" field parsed +from Metalink/XML files is trusted. When no Metalink/XML is available, +the mother URL is trusted. + +3.1.2 The final name +==================== + +The "Directory Options" are combined with the trusted name. The result +is evaluated again by the '1. Security features'. If the path is found +unsafe, only the basename of the final name is considered. If this is +found unsafe too, the file is not downloaded. 4. Metalink/XML *************** diff --git a/src/metalink.c b/src/metalink.c index be723dc3..721236f2 100644 --- a/src/metalink.c +++ b/src/metalink.c @@ -69,7 +69,16 @@ retrieve_from_metalink (const metalink_t* metalink) bool _output_stream_regular = output_stream_regular; char *_output_document = opt.output_document; - DEBUGP (("Retrieving from Metalink\n")); + /* metalink file counter */ + unsigned mfc = 0; + + /* metalink retrieval type */ + const char *metatpy = metalink->origin ? "Metalink/HTTP" : "Metalink/XML"; + + /* metalink mother source */ + char *metasrc = metalink->origin ? metalink->origin : opt.input_metalink; + + DEBUGP (("Retrieving from Metalink %s\n", quote (metasrc))); /* No files to download. */ if (!metalink->files) @@ -87,6 +96,8 @@ retrieve_from_metalink (const metalink_t* metalink) { metalink_file_t *mfile = *mfile_ptr; metalink_resource_t **mres_ptr; + char *planname = NULL; + char *trsrname = NULL; char *filename = NULL; char *basename = NULL; char *safename = NULL; @@ -104,30 +115,76 @@ retrieve_from_metalink (const metalink_t* metalink) output_stream = NULL; + mfc++; + /* The directory prefix for opt.metalink_over_http is handled by src/url.c (url_file_name), do not add it a second time. */ if (!metalink->origin && opt.dir_prefix && strlen (opt.dir_prefix)) - filename = aprintf ("%s/%s", opt.dir_prefix, mfile->name); + planname = aprintf ("%s/%s", opt.dir_prefix, mfile->name); else - filename = xstrdup (mfile->name); + planname = xstrdup (mfile->name); - DEBUGP (("Processing metalink file %s...\n", quote (mfile->name))); + /* With Metalink/HTTP, trust the metalink file name (from cli). + With --trust-server-names, trust the Metalink/XML file name, + otherwise, use the basename of --input-metalink followed by + the metalink file counter as suffix. */ + if (metalink->origin || opt.trustservernames) + { + trsrname = xstrdup (mfile->name); + } + else + { + trsrname = xstrdup (get_metalink_basename (opt.input_metalink)); + append_suffix_number (&trsrname, ".#", mfc); + } + + /* Add the directory prefix for opt.input_metalink. */ + if (!metalink->origin && opt.dir_prefix && strlen (opt.dir_prefix)) + filename = aprintf ("%s/%s", opt.dir_prefix, trsrname); + else + filename = xstrdup (trsrname); /* Enforce libmetalink's metalink_check_safe_path(). */ basename = get_metalink_basename (filename); safename = metalink_check_safe_path (filename) ? filename : basename; - if (filename != safename) - logprintf (LOG_NOTQUIET, - _("Unsafe metalink file %s. Stripping directory...\n"), - quote (filename)); + DEBUGP (("Processing metalink file %s...\n", quote (mfile->name))); + DEBUGP (("\n")); + DEBUGP ((" %s\n", metatpy)); + DEBUGP (("\n")); + DEBUGP ((" --trust-server-names %s\n", opt.trustservernames ? "true" : "false")); + DEBUGP ((" --directory-prefix %s\n", quote (opt.dir_prefix ? opt.dir_prefix : ""))); + DEBUGP (("\n")); + DEBUGP ((" Counted metalink file %u\n", mfc)); + DEBUGP ((" Planned metalink file %s\n", quote (planname ? planname : ""))); + DEBUGP ((" Trusted metalink file %s\n", quote (trsrname ? trsrname : ""))); + DEBUGP ((" Current metalink file %s\n", quote (filename ? filename : ""))); + DEBUGP ((" Cleaned metalink file %s\n", quote (basename ? basename : ""))); + DEBUGP ((" Secured metalink file %s\n", quote (safename ? safename : ""))); + DEBUGP (("\n")); - if (!basename) + /* Verify if the planned metalink file name is safe. */ + if (!safename || strcmp (planname, safename)) { logprintf (LOG_NOTQUIET, - _("Rejecting metalink file. Invalid basename.\n")); - xfree (filename); - continue; + _("[--trust-server-names %s, --directory-prefix=%s]\n"), + (opt.trustservernames ? "true" : "false"), + quote (opt.dir_prefix ? opt.dir_prefix : "")); + logprintf (LOG_NOTQUIET, + _("Planned metalink file: %s\n"), + quote (planname ? planname : "")); + logprintf (LOG_NOTQUIET, + _("Secured metalink file: %s\n"), + quote (safename ? safename : "")); + if (!safename) + { + logprintf (LOG_NOTQUIET, + _("Rejecting metalink file. Unsafe name.\n")); + xfree (planname); + xfree (trsrname); + xfree (filename); + continue; + } } /* Resources are sorted by priority. */ @@ -190,12 +247,6 @@ retrieve_from_metalink (const metalink_t* metalink) /* Avoid recursive Metalink from HTTP headers. */ bool _metalink_http = opt.metalink_over_http; - /* FIXME: could be useless. */ - if (strcmp (url->file, basename)) - logprintf (LOG_VERBOSE, - _("URL file name %s and Metalink file name %s are different.\n"), - quote_n (0, url->file), quote_n (1, basename)); - /* If output_stream is not NULL, then we have failed on previous resource and are retrying. Thus, continue with the next resource. Do not close output_stream @@ -662,6 +713,8 @@ gpg_skip_verification: } xfree (destname); xfree (filename); + xfree (trsrname); + xfree (planname); } /* Iterate over files. */ /* Restore original values. */ @@ -700,6 +753,22 @@ get_metalink_basename (char *name) return metalink_check_safe_path (basename) ? basename : NULL; } +/* + Append a separator and a numeric suffix to a string. + + The string is permanently modified. +*/ +void +append_suffix_number (char **str, const char *sep, wgint num) +{ + char *new, buf[24]; + + number_to_string (buf, num); + new = aprintf ("%s%s%s", *str ? *str : "", sep ? sep : "", buf); + xfree (*str); + *str = new; +} + /* Append the suffix ".badhash" to the file NAME, except without overwriting an existing file with that name and suffix. */ void diff --git a/src/metalink.h b/src/metalink.h index d052b70d..c9dd73ed 100644 --- a/src/metalink.h +++ b/src/metalink.h @@ -51,6 +51,7 @@ int metalink_check_safe_path (const char *path); char *last_component (char const *name); char *get_metalink_basename (char *name); +void append_suffix_number (char **str, const char *sep, wgint num); void badhash_suffix (char *name); void badhash_or_remove (char *name); diff --git a/testenv/Makefile.am b/testenv/Makefile.am index f14be076..f8c3da93 100644 --- a/testenv/Makefile.am +++ b/testenv/Makefile.am @@ -32,7 +32,19 @@ if METALINK_IS_ENABLED Test-metalink-xml-continue.py \ Test-metalink-xml-relpath.py \ Test-metalink-xml-abspath.py \ - Test-metalink-xml-homepath.py + Test-metalink-xml-homepath.py \ + Test-metalink-xml-trust.py \ + Test-metalink-xml-relpath-trust.py \ + Test-metalink-xml-abspath-trust.py \ + Test-metalink-xml-homepath-trust.py \ + Test-metalink-xml-prefix.py \ + Test-metalink-xml-relprefix.py \ + Test-metalink-xml-absprefix.py \ + Test-metalink-xml-homeprefix.py \ + Test-metalink-xml-prefix-trust.py \ + Test-metalink-xml-relprefix-trust.py \ + Test-metalink-xml-absprefix-trust.py \ + Test-metalink-xml-homeprefix-trust.py else METALINK_TESTS = endif diff --git a/testenv/Test-metalink-xml-abspath-trust.py b/testenv/Test-metalink-xml-abspath-trust.py new file mode 100755 index 00000000..35354d00 --- /dev/null +++ b/testenv/Test-metalink-xml-abspath-trust.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import hashlib + +""" + This is to test if Metalink/XML forbids absolute paths. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" +############# File Definitions ############################################### +bad = "Ouch!" + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +MetaXml = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + +""" + +wrong_file = WgetFile ("wrong_file", bad) + +# rejected by libmetalink +File1_orig = WgetFile ("File1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +File2_orig = WgetFile ("File2", File2) +File2_down = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +MetaFile = WgetFile ("test.metalink", MetaXml) + +WGET_OPTIONS = "--trust-server-names --input-metalink test.metalink" +WGET_URLS = [[]] + +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, +]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + File2_down, + MetaFile +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) +MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml-abspath.py b/testenv/Test-metalink-xml-abspath.py index 4447b0e3..85d544b5 100755 --- a/testenv/Test-metalink-xml-abspath.py +++ b/testenv/Test-metalink-xml-abspath.py @@ -5,12 +5,27 @@ from misc.wget_file import WgetFile import hashlib """ - This is to test if Metalink XML file escapes current directory. + This is to test if Metalink/XML forbids absolute paths. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. """ ############# File Definitions ############################################### +bad = "Ouch!" + File1 = "Would you like some Tea?" File1_lowPref = "Do not take this" File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + MetaXml = \ """ @@ -21,36 +36,63 @@ MetaXml = \ GNU GPL http://www.gnu.org/licenses/gpl.html - Wget Test File 1 + Wget Test Files 1.2.3 - Wget Test File 1 description + Wget Test Files description - + {{FILE1_HASH}} - http://broken.example/File1 + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + """ -A_File = WgetFile ("File1", File1) -B_File = WgetFile ("File1_lowPref", File1_lowPref) +wrong_file = WgetFile ("wrong_file", bad) + +# rejected by libmetalink +File1_orig = WgetFile ("File1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +File2_orig = WgetFile ("File2", File2) +File2_down = WgetFile ("test.metalink.#1", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + MetaFile = WgetFile ("test.metalink", MetaXml) WGET_OPTIONS = "--input-metalink test.metalink" WGET_URLS = [[]] -Files = [[A_File, B_File]] +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono +]] Existing_Files = [MetaFile] ExpectedReturnCode = 0 -ExpectedDownloadedFiles = [MetaFile] +ExpectedDownloadedFiles = [ + File2_down, + MetaFile +] ################ Pre and Post Test Hooks ##################################### pre_test = { @@ -77,6 +119,7 @@ http_test.server_setup() srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) MetaFile.content = MetaXml diff --git a/testenv/Test-metalink-xml-absprefix-trust.py b/testenv/Test-metalink-xml-absprefix-trust.py new file mode 100755 index 00000000..1cb46ec4 --- /dev/null +++ b/testenv/Test-metalink-xml-absprefix-trust.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import hashlib + +""" + This is to test Metalink/XML absolute directory prefix support in Wget. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" +############# File Definitions ############################################### +bad = "Ouch!" + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + +MetaXml = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + + + +""" + +wrong_file = WgetFile ("wrong_file", bad) + +File1_orig = WgetFile ("File1", File1) +File1_down = WgetFile ("File1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +# rejected by libmetalink +File2_orig = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +# rejected by libmetalink +File4_orig = WgetFile ("File4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("File5", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + +MetaFile = WgetFile ("test.metalink", MetaXml) + +WGET_OPTIONS = "--trust-server-names --directory-prefix /dir --input-metalink test.metalink" +WGET_URLS = [[]] + +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + File1_down, + File5_down, + MetaFile +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) +MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) +MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml-absprefix.py b/testenv/Test-metalink-xml-absprefix.py new file mode 100755 index 00000000..39423d3a --- /dev/null +++ b/testenv/Test-metalink-xml-absprefix.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import hashlib + +""" + This is to test Metalink/XML absolute directory prefix support in Wget. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" +############# File Definitions ############################################### +bad = "Ouch!" + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + +MetaXml = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + + + +""" + +wrong_file = WgetFile ("wrong_file", bad) + +File1_orig = WgetFile ("File1", File1) +File1_down = WgetFile ("test.metalink.#1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +# rejected by libmetalink +File2_orig = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +# rejected by libmetalink +File4_orig = WgetFile ("File4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("test.metalink.#2", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + +MetaFile = WgetFile ("test.metalink", MetaXml) + +WGET_OPTIONS = "--directory-prefix /dir --input-metalink test.metalink" +WGET_URLS = [[]] + +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + File1_down, + File5_down, + MetaFile +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) +MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) +MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml-continue.py b/testenv/Test-metalink-xml-continue.py index 8e3c40ae..068eb379 100644 --- a/testenv/Test-metalink-xml-continue.py +++ b/testenv/Test-metalink-xml-continue.py @@ -6,6 +6,14 @@ import hashlib """ This is to test Metalink/XML --continue support in Wget. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. """ ############# File Definitions ############################################### bad = "Ouch!" @@ -98,24 +106,24 @@ MetaXml = \ wrong_file = WgetFile ("wrong_file", bad) # partial File1_down to continue -File0_part = WgetFile ("File1", File0) +File0_part = WgetFile ("test.metalink.#1", File0) File1_orig = WgetFile ("File1", File1) -File1_down = WgetFile ("File1", File1) +File1_down = WgetFile ("test.metalink.#1", File1) File1_nono = WgetFile ("File1_lowPref", File1_lowPref) # no good resources on purpose, this file shall be kept -File2_ouch = WgetFile ("File2", bad) +File2_ouch = WgetFile ("test.metalink.#2", bad) File3_orig = WgetFile ("File3", File3) -File3_down = WgetFile ("File3", File3) +File3_down = WgetFile ("test.metalink.#3", File3) File3_nono = WgetFile ("File3_lowPref", File3_lowPref) # no good resources on purpose, this file shall be kept -File4_ouch = WgetFile ("File4", bad) +File4_ouch = WgetFile ("test.metalink.#4", bad) File5_orig = WgetFile ("File5", File5) -File5_down = WgetFile ("File5", File5) +File5_down = WgetFile ("test.metalink.#5", File5) File5_nono = WgetFile ("File5_lowPref", File5_lowPref) MetaFile = WgetFile ("test.metalink", MetaXml) diff --git a/testenv/Test-metalink-xml-homepath-trust.py b/testenv/Test-metalink-xml-homepath-trust.py new file mode 100755 index 00000000..e072c819 --- /dev/null +++ b/testenv/Test-metalink-xml-homepath-trust.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import hashlib + +""" + This is to test if Metalink/XML forbids the home path and names + beginning with the ~ (tilde) character. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" +############# File Definitions ############################################### +bad = "Ouch!" + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + +MetaXml = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + + + +""" + +wrong_file = WgetFile ("wrong_file", bad) + +# rejected by libmetalink +File1_orig = WgetFile ("File1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +# rejected by libmetalink +File2_orig = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +File4_orig = WgetFile ("File4", File4) +File4_down = WgetFile ("dir/File4~", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("dir/~/File5", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + +MetaFile = WgetFile ("test.metalink", MetaXml) + +WGET_OPTIONS = "--trust-server-names --input-metalink test.metalink" +WGET_URLS = [[]] + +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + File4_down, + File5_down, + MetaFile +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) +MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) +MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml-homepath.py b/testenv/Test-metalink-xml-homepath.py index aa06f2be..6e2e9de4 100755 --- a/testenv/Test-metalink-xml-homepath.py +++ b/testenv/Test-metalink-xml-homepath.py @@ -5,12 +5,40 @@ from misc.wget_file import WgetFile import hashlib """ - This is to test if Metalink XML file escapes current directory. + This is to test if Metalink/XML forbids the home path and names + beginning with the ~ (tilde) character. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. """ ############# File Definitions ############################################### +bad = "Ouch!" + File1 = "Would you like some Tea?" File1_lowPref = "Do not take this" File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + MetaXml = \ """ @@ -21,36 +49,112 @@ MetaXml = \ GNU GPL http://www.gnu.org/licenses/gpl.html - Wget Test File 1 + Wget Test Files 1.2.3 - Wget Test File 1 description + Wget Test Files description - + {{FILE1_HASH}} - http://broken.example/File1 + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + """ -A_File = WgetFile ("File1", File1) -B_File = WgetFile ("File1_lowPref", File1_lowPref) +wrong_file = WgetFile ("wrong_file", bad) + +# rejected by libmetalink +File1_orig = WgetFile ("File1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +# rejected by libmetalink +File2_orig = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +File4_orig = WgetFile ("File4", File4) +File4_down = WgetFile ("test.metalink.#1", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("test.metalink.#2", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + MetaFile = WgetFile ("test.metalink", MetaXml) WGET_OPTIONS = "--input-metalink test.metalink" WGET_URLS = [[]] -Files = [[A_File, B_File]] +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] Existing_Files = [MetaFile] ExpectedReturnCode = 0 -ExpectedDownloadedFiles = [MetaFile] +ExpectedDownloadedFiles = [ + File4_down, + File5_down, + MetaFile +] ################ Pre and Post Test Hooks ##################################### pre_test = { @@ -77,6 +181,10 @@ http_test.server_setup() srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) MetaFile.content = MetaXml diff --git a/testenv/Test-metalink-xml-homeprefix-trust.py b/testenv/Test-metalink-xml-homeprefix-trust.py new file mode 100755 index 00000000..a74243ee --- /dev/null +++ b/testenv/Test-metalink-xml-homeprefix-trust.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import hashlib + +""" + This is to test Metalink/XML home directory prefix support in Wget. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" +############# File Definitions ############################################### +bad = "Ouch!" + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + +MetaXml = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + + + +""" + +wrong_file = WgetFile ("wrong_file", bad) + +File1_orig = WgetFile ("File1", File1) +File1_down = WgetFile ("File1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +# rejected by libmetalink +File2_orig = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +# rejected by libmetalink +File4_orig = WgetFile ("File4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("File5", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + +MetaFile = WgetFile ("test.metalink", MetaXml) + +WGET_OPTIONS = "--trust-server-names --directory-prefix ~/dir --input-metalink test.metalink" +WGET_URLS = [[]] + +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + File1_down, + File5_down, + MetaFile +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) +MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) +MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml-homeprefix.py b/testenv/Test-metalink-xml-homeprefix.py new file mode 100755 index 00000000..dc1df327 --- /dev/null +++ b/testenv/Test-metalink-xml-homeprefix.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import hashlib + +""" + This is to test Metalink/XML home directory prefix support in Wget. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" +############# File Definitions ############################################### +bad = "Ouch!" + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + +MetaXml = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + + + +""" + +wrong_file = WgetFile ("wrong_file", bad) + +File1_orig = WgetFile ("File1", File1) +File1_down = WgetFile ("test.metalink.#1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +# rejected by libmetalink +File2_orig = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +# rejected by libmetalink +File4_orig = WgetFile ("File4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("test.metalink.#2", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + +MetaFile = WgetFile ("test.metalink", MetaXml) + +WGET_OPTIONS = "--directory-prefix ~/dir --input-metalink test.metalink" +WGET_URLS = [[]] + +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + File1_down, + File5_down, + MetaFile +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) +MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) +MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml-prefix-trust.py b/testenv/Test-metalink-xml-prefix-trust.py new file mode 100755 index 00000000..b9710064 --- /dev/null +++ b/testenv/Test-metalink-xml-prefix-trust.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import hashlib + +""" + This is to test Metalink/XML relative directory prefix support in Wget. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" +############# File Definitions ############################################### +bad = "Ouch!" + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + +MetaXml = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + + + +""" + +wrong_file = WgetFile ("wrong_file", bad) + +File1_orig = WgetFile ("File1", File1) +File1_down = WgetFile ("dir/subdir/File1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +# rejected by libmetalink +File2_orig = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +# rejected by libmetalink +File4_orig = WgetFile ("File4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("dir/subdir/File5", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + +MetaFile = WgetFile ("test.metalink", MetaXml) + +WGET_OPTIONS = "--trust-server-names --directory-prefix dir --input-metalink test.metalink" +WGET_URLS = [[]] + +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + File1_down, + File5_down, + MetaFile +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) +MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) +MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml-prefix.py b/testenv/Test-metalink-xml-prefix.py new file mode 100755 index 00000000..c94e53a6 --- /dev/null +++ b/testenv/Test-metalink-xml-prefix.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import hashlib + +""" + This is to test Metalink/XML relative directory prefix support in Wget. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" +############# File Definitions ############################################### +bad = "Ouch!" + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + +MetaXml = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + + + +""" + +wrong_file = WgetFile ("wrong_file", bad) + +File1_orig = WgetFile ("File1", File1) +File1_down = WgetFile ("dir/test.metalink.#1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +# rejected by libmetalink +File2_orig = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +# rejected by libmetalink +File4_orig = WgetFile ("File4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("dir/test.metalink.#2", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + +MetaFile = WgetFile ("test.metalink", MetaXml) + +WGET_OPTIONS = "--directory-prefix dir --input-metalink test.metalink" +WGET_URLS = [[]] + +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + File1_down, + File5_down, + MetaFile +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) +MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) +MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml-relpath-trust.py b/testenv/Test-metalink-xml-relpath-trust.py new file mode 100755 index 00000000..f5a3a3bb --- /dev/null +++ b/testenv/Test-metalink-xml-relpath-trust.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import hashlib + +""" + This is to test if Metalink/XML forbids relative paths. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" +############# File Definitions ############################################### +bad = "Ouch!" + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + +MetaXml = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + + + +""" + +wrong_file = WgetFile ("wrong_file", bad) + +# rejected by libmetalink +File1_orig = WgetFile ("File1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +# rejected by libmetalink +File2_orig = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +# rejected by libmetalink +File4_orig = WgetFile ("File4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("File5", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + +MetaFile = WgetFile ("test.metalink", MetaXml) + +WGET_OPTIONS = "--trust-server-names --input-metalink test.metalink" +WGET_URLS = [[]] + +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + File5_down, + MetaFile +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) +MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) +MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml-relpath.py b/testenv/Test-metalink-xml-relpath.py index 9790cd40..7d9307a7 100755 --- a/testenv/Test-metalink-xml-relpath.py +++ b/testenv/Test-metalink-xml-relpath.py @@ -5,12 +5,39 @@ from misc.wget_file import WgetFile import hashlib """ - This is to test if Metalink XML file escapes current directory. + This is to test if Metalink/XML forbids relative paths. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. """ ############# File Definitions ############################################### +bad = "Ouch!" + File1 = "Would you like some Tea?" File1_lowPref = "Do not take this" File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + MetaXml = \ """ @@ -21,76 +48,111 @@ MetaXml = \ GNU GPL http://www.gnu.org/licenses/gpl.html - Wget Test File 1 + Wget Test Files 1.2.3 - Wget Test File 1 description + Wget Test Files description - + {{FILE1_HASH}} - http://{{SRV_HOST}}:{{SRV_PORT}}/File1 - - - - - {{FILE1_HASH}} - - - http://{{SRV_HOST}}:{{SRV_PORT}}/File1 - - - - - {{FILE1_HASH}} - - - http://{{SRV_HOST}}:{{SRV_PORT}}/File1 - - - - - {{FILE1_HASH}} - - - http://{{SRV_HOST}}:{{SRV_PORT}}/File1 - - - - - {{FILE1_HASH}} - - - http://{{SRV_HOST}}:{{SRV_PORT}}/File1 - - - - - {{FILE1_HASH}} - - - http://broken.example/File1 + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + """ -A_File = WgetFile ("File1", File1) -B_File = WgetFile ("File1_lowPref", File1_lowPref) +wrong_file = WgetFile ("wrong_file", bad) + +# rejected by libmetalink +File1_orig = WgetFile ("File1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +# rejected by libmetalink +File2_orig = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +# rejected by libmetalink +File4_orig = WgetFile ("File4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("test.metalink.#1", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + MetaFile = WgetFile ("test.metalink", MetaXml) WGET_OPTIONS = "--input-metalink test.metalink" WGET_URLS = [[]] -Files = [[A_File, B_File]] +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] Existing_Files = [MetaFile] ExpectedReturnCode = 0 -ExpectedDownloadedFiles = [WgetFile ("dir/subdir/File1", File1), MetaFile] +ExpectedDownloadedFiles = [ + File5_down, + MetaFile +] ################ Pre and Post Test Hooks ##################################### pre_test = { @@ -117,6 +179,10 @@ http_test.server_setup() srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) MetaFile.content = MetaXml diff --git a/testenv/Test-metalink-xml-relprefix-trust.py b/testenv/Test-metalink-xml-relprefix-trust.py new file mode 100755 index 00000000..2d6344d0 --- /dev/null +++ b/testenv/Test-metalink-xml-relprefix-trust.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import hashlib + +""" + This is to test Metalink/XML relative directory prefix support in Wget. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" +############# File Definitions ############################################### +bad = "Ouch!" + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + +MetaXml = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + + + +""" + +wrong_file = WgetFile ("wrong_file", bad) + +File1_orig = WgetFile ("File1", File1) +File1_down = WgetFile ("File1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +# rejected by libmetalink +File2_orig = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +# rejected by libmetalink +File4_orig = WgetFile ("File4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("File5", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + +MetaFile = WgetFile ("test.metalink", MetaXml) + +WGET_OPTIONS = "--trust-server-names --directory-prefix ../dir --input-metalink test.metalink" +WGET_URLS = [[]] + +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + File1_down, + File5_down, + MetaFile +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) +MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) +MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml-relprefix.py b/testenv/Test-metalink-xml-relprefix.py new file mode 100755 index 00000000..35883d8e --- /dev/null +++ b/testenv/Test-metalink-xml-relprefix.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import hashlib + +""" + This is to test Metalink/XML relative directory prefix support in Wget. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" +############# File Definitions ############################################### +bad = "Ouch!" + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + +MetaXml = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + + + +""" + +wrong_file = WgetFile ("wrong_file", bad) + +File1_orig = WgetFile ("File1", File1) +File1_down = WgetFile ("test.metalink.#1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +# rejected by libmetalink +File2_orig = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +# rejected by libmetalink +File4_orig = WgetFile ("File4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("test.metalink.#2", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + +MetaFile = WgetFile ("test.metalink", MetaXml) + +WGET_OPTIONS = "--directory-prefix ../dir --input-metalink test.metalink" +WGET_URLS = [[]] + +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + File1_down, + File5_down, + MetaFile +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) +MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) +MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml-trust.py b/testenv/Test-metalink-xml-trust.py new file mode 100755 index 00000000..c930c79c --- /dev/null +++ b/testenv/Test-metalink-xml-trust.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import hashlib + +""" + This is to test Metalink/XML file support in Wget. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" +############# File Definitions ############################################### +bad = "Ouch!" + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + +MetaXml = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + + + +""" + +wrong_file = WgetFile ("wrong_file", bad) + +File1_orig = WgetFile ("File1", File1) +File1_down = WgetFile ("File1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +File2_orig = WgetFile ("File2", File2) +File2_down = WgetFile ("File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +File3_orig = WgetFile ("File3", File3) +File3_down = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +File4_orig = WgetFile ("File4", File4) +File4_down = WgetFile ("File4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("File5", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + +MetaFile = WgetFile ("test.metalink", MetaXml) + +WGET_OPTIONS = "--trust-server-names --input-metalink test.metalink" +WGET_URLS = [[]] + +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + File1_down, + File2_down, + File3_down, + File4_down, + File5_down, + MetaFile +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) +MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) +MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml.py b/testenv/Test-metalink-xml.py index fdeede64..3f1eafd4 100755 --- a/testenv/Test-metalink-xml.py +++ b/testenv/Test-metalink-xml.py @@ -5,12 +5,39 @@ from misc.wget_file import WgetFile import hashlib """ - This is to test Metalink as XML file support in Wget. + This is to test Metalink/XML file support in Wget. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. """ ############# File Definitions ############################################### +bad = "Ouch!" + File1 = "Would you like some Tea?" File1_lowPref = "Do not take this" File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + MetaXml = \ """ @@ -21,36 +48,115 @@ MetaXml = \ GNU GPL http://www.gnu.org/licenses/gpl.html - Wget Test File 1 + Wget Test Files 1.2.3 - Wget Test File 1 description + Wget Test Files description {{FILE1_HASH}} - http://broken.example/File1 + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + """ -A_File = WgetFile ("File1", File1) -B_File = WgetFile ("File1_lowPref", File1_lowPref) +wrong_file = WgetFile ("wrong_file", bad) + +File1_orig = WgetFile ("File1", File1) +File1_down = WgetFile ("test.metalink.#1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +File2_orig = WgetFile ("File2", File2) +File2_down = WgetFile ("test.metalink.#2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +File3_orig = WgetFile ("File3", File3) +File3_down = WgetFile ("test.metalink.#3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +File4_orig = WgetFile ("File4", File4) +File4_down = WgetFile ("test.metalink.#4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("test.metalink.#5", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + MetaFile = WgetFile ("test.metalink", MetaXml) WGET_OPTIONS = "--input-metalink test.metalink" WGET_URLS = [[]] -Files = [[A_File, B_File]] +Files = [[ + wrong_file, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] Existing_Files = [MetaFile] ExpectedReturnCode = 0 -ExpectedDownloadedFiles = [A_File, MetaFile] +ExpectedDownloadedFiles = [ + File1_down, + File2_down, + File3_down, + File4_down, + File5_down, + MetaFile +] ################ Pre and Post Test Hooks ##################################### pre_test = { @@ -77,6 +183,10 @@ http_test.server_setup() srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256) +MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256) +MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256) +MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256) +MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256) MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host) MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port)) MetaFile.content = MetaXml