mirror of
https://github.com/mirror/wget.git
synced 2025-03-14 11:50:18 +08:00
Merge remote-tracking branch 'origin/master' into parallel-wget
This commit is contained in:
Giuseppe Scrivano
commit
fec5bb9e93
@ -1,3 +1,7 @@
|
||||
2013-05-17 Bykov Aleksey <gnfalex@rambler.ru>
|
||||
|
||||
* bootstrap: Add `mkostemp'
|
||||
|
||||
2012-10-07 Giuseppe Scrivano <gscrivano@gnu.org>
|
||||
|
||||
* configure.ac: Check for patchconf.
|
||||
|
||||
@ -52,6 +52,7 @@ maintainer-makefile
|
||||
mbtowc
|
||||
mkdir
|
||||
mkstemp
|
||||
mkostemp
|
||||
crypto/md5
|
||||
crypto/sha1
|
||||
crypto/sha256
|
||||
|
||||
@ -1,3 +1,14 @@
|
||||
2013-06-17 Dave Reisner <dreisner@archlinux.org> (tiny change)
|
||||
|
||||
* texi2pod.pl: Fix formatting error that causes build to fail with
|
||||
Perl 5.18
|
||||
|
||||
2013-06-17 Darshit Shah <darnir@gmail.com>
|
||||
|
||||
* wget.texi (POST): Explain the new redirection rules.
|
||||
* wget.texi (Other HTTP Methods): Same.
|
||||
* wget.texi (body-data): Fix typo in description.
|
||||
|
||||
2013-05-10 Darshit Shah <darnir@gmail.com> (tiny change)
|
||||
|
||||
* wget.texi (No of tries): Fix typo to make it clear that --tries
|
||||
|
||||
@ -291,7 +291,7 @@ while(<$inf>) {
|
||||
if (defined $1) {
|
||||
my $thing = $1;
|
||||
if ($ic =~ /\@asis/) {
|
||||
$_ = "\n=item $thing\n";
|
||||
$_ = "\n=item C<$thing>\n";
|
||||
} else {
|
||||
# Entity escapes prevent munging by the <> processing below.
|
||||
$_ = "\n=item $ic\<$thing\>\n";
|
||||
|
||||
@ -1504,14 +1504,15 @@ use chunked unless it knows it's talking to an HTTP/1.1 server. And it
|
||||
can't know that until it receives a response, which in turn requires the
|
||||
request to have been completed -- a chicken-and-egg problem.
|
||||
|
||||
Note: if Wget is redirected after the POST request is completed, it
|
||||
will not send the POST data to the redirected URL. This is because
|
||||
URLs that process POST often respond with a redirection to a regular
|
||||
page, which does not desire or accept POST. It is not completely
|
||||
clear that this behavior is optimal; if it doesn't work out, it might
|
||||
be changed in the future.
|
||||
Note: As of version 1.15 if Wget is redirected after the POST request is
|
||||
completed, its behaviour will depend on the response code returned by the
|
||||
server. In case of a 301 Moved Permanently, 302 Moved Temporarily or
|
||||
307 Temporary Redirect, Wget will, in accordance with RFC2616, continue
|
||||
to send a POST request.
|
||||
In case a server wants the client to change the Request method upon
|
||||
redirection, it should send a 303 See Other response code.
|
||||
|
||||
This example shows how to log to a server using POST and then proceed to
|
||||
This example shows how to log in to a server using POST and then proceed to
|
||||
download the desired pages, presumably only accessible to authorized
|
||||
users:
|
||||
|
||||
@ -1544,8 +1545,8 @@ Method to the server.
|
||||
@item --body-data=@var{Data-String}
|
||||
@itemx --body-file=@var{Data-File}
|
||||
Must be set when additional data needs to be sent to the server along with the
|
||||
Method specified using @samp{--method}. @samp{--post-data} sends @var{string} as
|
||||
data, whereas @samp{--post-file} sends the contents of @var{file}. Other than that,
|
||||
Method specified using @samp{--method}. @samp{--body-data} sends @var{string} as
|
||||
data, whereas @samp{--body-file} sends the contents of @var{file}. Other than that,
|
||||
they work in exactly the same way.
|
||||
|
||||
Currently, @samp{--body-file} is @emph{not} for transmitting files as a whole.
|
||||
@ -1557,10 +1558,13 @@ BODY Data in advance, and hence the argument to @samp{--body-file} should be a
|
||||
regular file. See @samp{--post-file} for a more detailed explanation.
|
||||
Only one of @samp{--body-data} and @samp{--body-file} should be specified.
|
||||
|
||||
Wget handles these requests in the same way that it handles @samp{--post-data}
|
||||
and @samp{--post-file}. If you invoke Wget with @samp{--method=POST} and the server
|
||||
responds with a redirect request, then Wget will revert to a GET request during the
|
||||
redirection as is explained in @samp{--post-data}.
|
||||
If Wget is redirected after the request is completed, Wget will
|
||||
suspend the current method and send a GET request till the redirection
|
||||
is completed. This is true for all redirection response codes except
|
||||
307 Temporary Redirect which is used to explicitly specify that the
|
||||
request method should @emph{not} change. Another exception is when
|
||||
the method is set to @code{POST}, in which case the redirection rules
|
||||
specified under @samp{--post-data} are followed.
|
||||
|
||||
@cindex Content-Disposition
|
||||
@item --content-disposition
|
||||
|
||||
@ -22,6 +22,34 @@
|
||||
* http.c (gethttp): Do not register the persistent connection too
|
||||
early.
|
||||
|
||||
2013-06-22 Ángel González <keisial@gmail.com>
|
||||
|
||||
* init.c (cmd_string_uppercase): Rewrite function.
|
||||
|
||||
2013-06-19 Tim Ruehsen <tim.ruehsen@gmx.de>
|
||||
|
||||
* connect.c (socket_ip_address): zero out ip address structure to
|
||||
avoid access to uninitialized values by inet_ntop().
|
||||
* ftp.c (ftp_loop_internal): fix segfault caused by warc_tmp NULL pointer.
|
||||
|
||||
2013-06-13 Darshit Shah <darnir@gmail.com>
|
||||
|
||||
* http.c (gethttp): Follow RFC 2616 and httpbis specifications when
|
||||
handling redirections. Do not suspend the method on 301/302 redirects.
|
||||
(gethttp): If method if not GET, we do not intend to download
|
||||
anything.
|
||||
* main.c (main): Set spider mode when opt.method is HEAD. This will
|
||||
prevent Wget from downloading any file.
|
||||
* retr.c (SUSPEND_METHOD): Rename macro SUSPEND_POST_DATA to
|
||||
SUSPEND_METHOD to more accurately reflect its use. Similarly rename
|
||||
related variables.
|
||||
|
||||
2013-05-14 Bykov Aleksey <gnfalex@rambler.ru>
|
||||
|
||||
* warc.c (warc_tempfile): For fix "Could not open temporary WARC manifest
|
||||
file." issue in Win system force to use `mkostemp(filename, O_TEMPORARY)'
|
||||
instead of `mkstemp(filename)'. Thank to Angel Gonzalez for help.
|
||||
|
||||
2013-05-14 Tim Ruehsen <tim.ruehsen@gmx.de>
|
||||
|
||||
* cookies.c (cookie_jar_load): Replaced read_whole_file() by getline().
|
||||
|
||||
@ -559,6 +559,7 @@ socket_ip_address (int sock, ip_address *ip, int endpoint)
|
||||
if (ret < 0)
|
||||
return false;
|
||||
|
||||
memset(ip, 0, sizeof(ip_address));
|
||||
ip->family = sockaddr->sa_family;
|
||||
switch (sockaddr->sa_family)
|
||||
{
|
||||
|
||||
30
src/ftp.c
30
src/ftp.c
@ -1459,21 +1459,6 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
|
||||
|
||||
orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
|
||||
|
||||
/* For file RETR requests, we can write a WARC record.
|
||||
We record the file contents to a temporary file. */
|
||||
if (warc_enabled && (con->cmd & DO_RETR))
|
||||
{
|
||||
warc_tmp = warc_tempfile ();
|
||||
if (warc_tmp == NULL)
|
||||
return WARC_TMP_FOPENERR;
|
||||
|
||||
if (!con->proxy && con->csock != -1)
|
||||
{
|
||||
warc_ip = (ip_address *) alloca (sizeof (ip_address));
|
||||
socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
|
||||
}
|
||||
}
|
||||
|
||||
/* THE loop. */
|
||||
do
|
||||
{
|
||||
@ -1501,6 +1486,21 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
|
||||
con->cmd |= DO_CWD;
|
||||
}
|
||||
|
||||
/* For file RETR requests, we can write a WARC record.
|
||||
We record the file contents to a temporary file. */
|
||||
if (warc_enabled && (con->cmd & DO_RETR) && warc_tmp == NULL)
|
||||
{
|
||||
warc_tmp = warc_tempfile ();
|
||||
if (warc_tmp == NULL)
|
||||
return WARC_TMP_FOPENERR;
|
||||
|
||||
if (!con->proxy && con->csock != -1)
|
||||
{
|
||||
warc_ip = (ip_address *) alloca (sizeof (ip_address));
|
||||
socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
|
||||
}
|
||||
}
|
||||
|
||||
/* Decide whether or not to restart. */
|
||||
if (con->cmd & DO_LIST)
|
||||
restval = 0;
|
||||
|
||||
32
src/http.c
32
src/http.c
@ -2918,9 +2918,32 @@ read_header:
|
||||
|
||||
A 307 should be redirected using the same method,
|
||||
in other words, a POST should be preserved and not
|
||||
converted to a GET in that case. */
|
||||
if (statcode == HTTP_STATUS_TEMPORARY_REDIRECT)
|
||||
return NEWLOCATION_KEEP_POST;
|
||||
converted to a GET in that case.
|
||||
|
||||
With strict adherence to RFC2616, POST requests are not
|
||||
converted to a GET request on 301 Permanent Redirect
|
||||
or 302 Temporary Redirect.
|
||||
|
||||
A switch may be provided later based on the HTTPbis draft
|
||||
that allows clients to convert POST requests to GET
|
||||
requests on 301 and 302 response codes. */
|
||||
switch (statcode)
|
||||
{
|
||||
case HTTP_STATUS_TEMPORARY_REDIRECT:
|
||||
return NEWLOCATION_KEEP_POST;
|
||||
break;
|
||||
case HTTP_STATUS_MOVED_PERMANENTLY:
|
||||
if (opt.method && strcasecmp (opt.method, "post") != 0)
|
||||
return NEWLOCATION_KEEP_POST;
|
||||
break;
|
||||
case HTTP_STATUS_MOVED_TEMPORARILY:
|
||||
if (opt.method && strcasecmp (opt.method, "post") != 0)
|
||||
return NEWLOCATION_KEEP_POST;
|
||||
break;
|
||||
default:
|
||||
return NEWLOCATION;
|
||||
break;
|
||||
}
|
||||
return NEWLOCATION;
|
||||
}
|
||||
}
|
||||
@ -3028,7 +3051,8 @@ read_header:
|
||||
}
|
||||
|
||||
/* Return if we have no intention of further downloading. */
|
||||
if ((!(*dt & RETROKF) && !opt.content_on_error) || head_only)
|
||||
if ((!(*dt & RETROKF) && !opt.content_on_error) || head_only
|
||||
|| (opt.method && strcasecmp (opt.method, "get") != 0))
|
||||
{
|
||||
/* In case the caller cares to look... */
|
||||
hs->len = 0;
|
||||
|
||||
15
src/init.c
15
src/init.c
@ -980,15 +980,16 @@ cmd_string (const char *com, const char *val, void *place)
|
||||
static bool
|
||||
cmd_string_uppercase (const char *com, const char *val, void *place)
|
||||
{
|
||||
char *q;
|
||||
bool ret = cmd_string (com, val, place);
|
||||
q = *((char **) place);
|
||||
if (!ret || q == NULL)
|
||||
return false;
|
||||
char *q, **pstring;
|
||||
pstring = (char **)place;
|
||||
xfree_null (*pstring);
|
||||
|
||||
for ( ;*q; *q++)
|
||||
*q = c_toupper (*q);
|
||||
*pstring = xmalloc (strlen (val) + 1);
|
||||
|
||||
for (q = *pstring; *val; val++, q++)
|
||||
*q = c_toupper (*val);
|
||||
|
||||
*q = '\0';
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@ -1422,6 +1422,13 @@ for details.\n\n"));
|
||||
}
|
||||
}
|
||||
|
||||
/* Set various options as required for opt.method. */
|
||||
|
||||
/* When user specifies HEAD as the method, we do not wish to download any
|
||||
files. Hence, set wget to run in spider mode. */
|
||||
if (opt.method && strcasecmp (opt.method, "HEAD") == 0)
|
||||
setoptval ("spider", "1", "spider");
|
||||
|
||||
/* Convert post_data to body-data and post_file_name to body-file options.
|
||||
This is required so as to remove redundant code later on in gethttp().
|
||||
The --post-data and --post-file options may also be removed in
|
||||
|
||||
38
src/retr.c
38
src/retr.c
@ -700,23 +700,23 @@ calc_rate (wgint bytes, double secs, int *units)
|
||||
}
|
||||
|
||||
|
||||
#define SUSPEND_POST_DATA do { \
|
||||
post_data_suspended = true; \
|
||||
saved_post_data = opt.body_data; \
|
||||
saved_post_file_name = opt.body_file; \
|
||||
#define SUSPEND_METHOD do { \
|
||||
method_suspended = true; \
|
||||
saved_body_data = opt.body_data; \
|
||||
saved_body_file_name = opt.body_file; \
|
||||
saved_method = opt.method; \
|
||||
opt.body_data = NULL; \
|
||||
opt.body_file = NULL; \
|
||||
opt.method = NULL; \
|
||||
} while (0)
|
||||
|
||||
#define RESTORE_POST_DATA do { \
|
||||
if (post_data_suspended) \
|
||||
#define RESTORE_METHOD do { \
|
||||
if (method_suspended) \
|
||||
{ \
|
||||
opt.body_data = saved_post_data; \
|
||||
opt.body_file = saved_post_file_name; \
|
||||
opt.body_data = saved_body_data; \
|
||||
opt.body_file = saved_body_file_name; \
|
||||
opt.method = saved_method; \
|
||||
post_data_suspended = false; \
|
||||
method_suspended = false; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
@ -744,10 +744,10 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
|
||||
char *local_file;
|
||||
int redirection_count = 0;
|
||||
|
||||
bool post_data_suspended = false;
|
||||
char *saved_post_data = NULL;
|
||||
bool method_suspended = false;
|
||||
char *saved_body_data = NULL;
|
||||
char *saved_method = NULL;
|
||||
char *saved_post_file_name = NULL;
|
||||
char *saved_body_file_name = NULL;
|
||||
|
||||
/* If dt is NULL, use local storage. */
|
||||
if (!dt)
|
||||
@ -793,7 +793,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
|
||||
proxy, error);
|
||||
xfree (url);
|
||||
xfree (error);
|
||||
RESTORE_POST_DATA;
|
||||
RESTORE_METHOD;
|
||||
result = PROXERR;
|
||||
goto bail;
|
||||
}
|
||||
@ -802,7 +802,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
|
||||
logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy);
|
||||
url_free (proxy_url);
|
||||
xfree (url);
|
||||
RESTORE_POST_DATA;
|
||||
RESTORE_METHOD;
|
||||
result = PROXERR;
|
||||
goto bail;
|
||||
}
|
||||
@ -886,7 +886,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
|
||||
xfree (url);
|
||||
xfree (mynewloc);
|
||||
xfree (error);
|
||||
RESTORE_POST_DATA;
|
||||
RESTORE_METHOD;
|
||||
goto bail;
|
||||
}
|
||||
|
||||
@ -908,7 +908,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
|
||||
}
|
||||
xfree (url);
|
||||
xfree (mynewloc);
|
||||
RESTORE_POST_DATA;
|
||||
RESTORE_METHOD;
|
||||
result = WRONGCODE;
|
||||
goto bail;
|
||||
}
|
||||
@ -931,8 +931,8 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
|
||||
RFC2616 HTTP/1.1 introduces code 307 Temporary Redirect
|
||||
specifically to preserve the method of the request.
|
||||
*/
|
||||
if (result != NEWLOCATION_KEEP_POST && !post_data_suspended)
|
||||
SUSPEND_POST_DATA;
|
||||
if (result != NEWLOCATION_KEEP_POST && !method_suspended)
|
||||
SUSPEND_METHOD;
|
||||
|
||||
goto redirected;
|
||||
}
|
||||
@ -995,7 +995,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
|
||||
xfree (url);
|
||||
}
|
||||
|
||||
RESTORE_POST_DATA;
|
||||
RESTORE_METHOD;
|
||||
|
||||
bail:
|
||||
if (register_status)
|
||||
|
||||
11
src/warc.c
11
src/warc.c
@ -51,10 +51,16 @@ as that of the covered work. */
|
||||
|
||||
#ifndef WINDOWS
|
||||
#include <libgen.h>
|
||||
#else
|
||||
#include <fcntl.h>
|
||||
#endif
|
||||
|
||||
#include "warc.h"
|
||||
|
||||
#ifndef O_TEMPORARY
|
||||
#define O_TEMPORARY 0
|
||||
#endif
|
||||
|
||||
extern char *version_string;
|
||||
|
||||
/* Set by main in main.c */
|
||||
@ -1147,12 +1153,15 @@ warc_tempfile (void)
|
||||
if (path_search (filename, 100, opt.warc_tempdir, "wget", true) == -1)
|
||||
return NULL;
|
||||
|
||||
int fd = mkstemp (filename);
|
||||
int fd = mkostemp (filename, O_TEMPORARY);
|
||||
if (fd < 0)
|
||||
return NULL;
|
||||
|
||||
#if !O_TEMPORARY
|
||||
if (unlink (filename) < 0)
|
||||
return NULL;
|
||||
#endif
|
||||
|
||||
|
||||
return fdopen (fd, "wb+");
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user